Expose storage_prefix logic, and remove duplicate code (#9621)

* expose storage prefix generation, remove duplicate code

* remove more duplicate code

* clean up import

* fix io test

* remove slicing

* Update frame/support/src/storage/mod.rs

Co-authored-by: Guillaume Thiolliere <gui.thiolliere@gmail.com>

Co-authored-by: Guillaume Thiolliere <gui.thiolliere@gmail.com>

substrate/frame/support/src/storage/generator/double_map.rs

@@ -16,8 +16,8 @@
 // limitations under the License.
 
 use crate::{
-	hash::{ReversibleStorageHasher, StorageHasher, Twox128},
-	storage::{self, unhashed, KeyPrefixIterator, PrefixIterator, StorageAppend},
+	hash::{ReversibleStorageHasher, StorageHasher},
+	storage::{self, storage_prefix, unhashed, KeyPrefixIterator, PrefixIterator, StorageAppend},
 	Never,
 };
 use codec::{Decode, Encode, EncodeLike, FullCodec, FullEncode};
@@ -62,16 +62,8 @@ pub trait StorageDoubleMap<K1: FullEncode, K2: FullEncode, V: FullCodec> {
 	/// The full prefix; just the hash of `module_prefix` concatenated to the hash of
 	/// `storage_prefix`.
 	fn prefix_hash() -> Vec<u8> {
-		let module_prefix_hashed = Twox128::hash(Self::module_prefix());
-		let storage_prefix_hashed = Twox128::hash(Self::storage_prefix());
-
-		let mut result =
-			Vec::with_capacity(module_prefix_hashed.len() + storage_prefix_hashed.len());
-
-		result.extend_from_slice(&module_prefix_hashed[..]);
-		result.extend_from_slice(&storage_prefix_hashed[..]);
-
-		result
+		let result = storage_prefix(Self::module_prefix(), Self::storage_prefix());
+		result.to_vec()
 	}
 
 	/// Convert an optional value retrieved from storage to the type queried.
@@ -85,16 +77,12 @@ pub trait StorageDoubleMap<K1: FullEncode, K2: FullEncode, V: FullCodec> {
 	where
 		KArg1: EncodeLike<K1>,
 	{
-		let module_prefix_hashed = Twox128::hash(Self::module_prefix());
-		let storage_prefix_hashed = Twox128::hash(Self::storage_prefix());
+		let storage_prefix = storage_prefix(Self::module_prefix(), Self::storage_prefix());
 		let key_hashed = k1.borrow().using_encoded(Self::Hasher1::hash);
 
-		let mut final_key = Vec::with_capacity(
-			module_prefix_hashed.len() + storage_prefix_hashed.len() + key_hashed.as_ref().len(),
-		);
+		let mut final_key = Vec::with_capacity(storage_prefix.len() + key_hashed.as_ref().len());
 
-		final_key.extend_from_slice(&module_prefix_hashed[..]);
-		final_key.extend_from_slice(&storage_prefix_hashed[..]);
+		final_key.extend_from_slice(&storage_prefix);
 		final_key.extend_from_slice(key_hashed.as_ref());
 
 		final_key
@@ -106,20 +94,15 @@ pub trait StorageDoubleMap<K1: FullEncode, K2: FullEncode, V: FullCodec> {
 		KArg1: EncodeLike<K1>,
 		KArg2: EncodeLike<K2>,
 	{
-		let module_prefix_hashed = Twox128::hash(Self::module_prefix());
-		let storage_prefix_hashed = Twox128::hash(Self::storage_prefix());
+		let storage_prefix = storage_prefix(Self::module_prefix(), Self::storage_prefix());
 		let key1_hashed = k1.borrow().using_encoded(Self::Hasher1::hash);
 		let key2_hashed = k2.borrow().using_encoded(Self::Hasher2::hash);
 
 		let mut final_key = Vec::with_capacity(
-			module_prefix_hashed.len() +
-				storage_prefix_hashed.len() +
-				key1_hashed.as_ref().len() +
-				key2_hashed.as_ref().len(),
+			storage_prefix.len() + key1_hashed.as_ref().len() + key2_hashed.as_ref().len(),
 		);
 
-		final_key.extend_from_slice(&module_prefix_hashed[..]);
-		final_key.extend_from_slice(&storage_prefix_hashed[..]);
+		final_key.extend_from_slice(&storage_prefix);
 		final_key.extend_from_slice(key1_hashed.as_ref());
 		final_key.extend_from_slice(key2_hashed.as_ref());
 
@@ -319,20 +302,16 @@ where
 		key2: KeyArg2,
 	) -> Option<V> {
 		let old_key = {
-			let module_prefix_hashed = Twox128::hash(Self::module_prefix());
-			let storage_prefix_hashed = Twox128::hash(Self::storage_prefix());
+			let storage_prefix = storage_prefix(Self::module_prefix(), Self::storage_prefix());
+
 			let key1_hashed = key1.borrow().using_encoded(OldHasher1::hash);
 			let key2_hashed = key2.borrow().using_encoded(OldHasher2::hash);
 
 			let mut final_key = Vec::with_capacity(
-				module_prefix_hashed.len() +
-					storage_prefix_hashed.len() +
-					key1_hashed.as_ref().len() +
-					key2_hashed.as_ref().len(),
+				storage_prefix.len() + key1_hashed.as_ref().len() + key2_hashed.as_ref().len(),
 			);
 
-			final_key.extend_from_slice(&module_prefix_hashed[..]);
-			final_key.extend_from_slice(&storage_prefix_hashed[..]);
+			final_key.extend_from_slice(&storage_prefix);
 			final_key.extend_from_slice(key1_hashed.as_ref());
 			final_key.extend_from_slice(key2_hashed.as_ref());
 

substrate/frame/support/src/storage/generator/map.rs

@@ -16,8 +16,8 @@
 // limitations under the License.
 
 use crate::{
-	hash::{ReversibleStorageHasher, StorageHasher, Twox128},
-	storage::{self, unhashed, KeyPrefixIterator, PrefixIterator, StorageAppend},
+	hash::{ReversibleStorageHasher, StorageHasher},
+	storage::{self, storage_prefix, unhashed, KeyPrefixIterator, PrefixIterator, StorageAppend},
 	Never,
 };
 use codec::{Decode, Encode, EncodeLike, FullCodec, FullEncode};
@@ -52,16 +52,8 @@ pub trait StorageMap<K: FullEncode, V: FullCodec> {
 	/// The full prefix; just the hash of `module_prefix` concatenated to the hash of
 	/// `storage_prefix`.
 	fn prefix_hash() -> Vec<u8> {
-		let module_prefix_hashed = Twox128::hash(Self::module_prefix());
-		let storage_prefix_hashed = Twox128::hash(Self::storage_prefix());
-
-		let mut result =
-			Vec::with_capacity(module_prefix_hashed.len() + storage_prefix_hashed.len());
-
-		result.extend_from_slice(&module_prefix_hashed[..]);
-		result.extend_from_slice(&storage_prefix_hashed[..]);
-
-		result
+		let result = storage_prefix(Self::module_prefix(), Self::storage_prefix());
+		result.to_vec()
 	}
 
 	/// Convert an optional value retrieved from storage to the type queried.
@@ -75,16 +67,12 @@ pub trait StorageMap<K: FullEncode, V: FullCodec> {
 	where
 		KeyArg: EncodeLike<K>,
 	{
-		let module_prefix_hashed = Twox128::hash(Self::module_prefix());
-		let storage_prefix_hashed = Twox128::hash(Self::storage_prefix());
+		let storage_prefix = storage_prefix(Self::module_prefix(), Self::storage_prefix());
 		let key_hashed = key.borrow().using_encoded(Self::Hasher::hash);
 
-		let mut final_key = Vec::with_capacity(
-			module_prefix_hashed.len() + storage_prefix_hashed.len() + key_hashed.as_ref().len(),
-		);
+		let mut final_key = Vec::with_capacity(storage_prefix.len() + key_hashed.as_ref().len());
 
-		final_key.extend_from_slice(&module_prefix_hashed[..]);
-		final_key.extend_from_slice(&storage_prefix_hashed[..]);
+		final_key.extend_from_slice(&storage_prefix);
 		final_key.extend_from_slice(key_hashed.as_ref());
 
 		final_key
@@ -330,18 +318,13 @@ impl<K: FullEncode, V: FullCodec, G: StorageMap<K, V>> storage::StorageMap<K, V>
 
 	fn migrate_key<OldHasher: StorageHasher, KeyArg: EncodeLike<K>>(key: KeyArg) -> Option<V> {
 		let old_key = {
-			let module_prefix_hashed = Twox128::hash(Self::module_prefix());
-			let storage_prefix_hashed = Twox128::hash(Self::storage_prefix());
+			let storage_prefix = storage_prefix(Self::module_prefix(), Self::storage_prefix());
 			let key_hashed = key.borrow().using_encoded(OldHasher::hash);
 
-			let mut final_key = Vec::with_capacity(
-				module_prefix_hashed.len() +
-					storage_prefix_hashed.len() +
-					key_hashed.as_ref().len(),
-			);
+			let mut final_key =
+				Vec::with_capacity(storage_prefix.len() + key_hashed.as_ref().len());
 
-			final_key.extend_from_slice(&module_prefix_hashed[..]);
-			final_key.extend_from_slice(&storage_prefix_hashed[..]);
+			final_key.extend_from_slice(&storage_prefix);
 			final_key.extend_from_slice(key_hashed.as_ref());
 
 			final_key

substrate/frame/support/src/storage/generator/nmap.rs

@@ -30,9 +30,8 @@
 //! be compromised.
 
 use crate::{
-	hash::{StorageHasher, Twox128},
 	storage::{
-		self,
+		self, storage_prefix,
 		types::{
 			EncodeLikeTuple, HasKeyPrefix, HasReversibleKeyPrefix, KeyGenerator,
 			ReversibleKeyGenerator, TupleToEncodedIter,
@@ -71,16 +70,8 @@ pub trait StorageNMap<K: KeyGenerator, V: FullCodec> {
 	/// The full prefix; just the hash of `module_prefix` concatenated to the hash of
 	/// `storage_prefix`.
 	fn prefix_hash() -> Vec<u8> {
-		let module_prefix_hashed = Twox128::hash(Self::module_prefix());
-		let storage_prefix_hashed = Twox128::hash(Self::storage_prefix());
-
-		let mut result =
-			Vec::with_capacity(module_prefix_hashed.len() + storage_prefix_hashed.len());
-
-		result.extend_from_slice(&module_prefix_hashed[..]);
-		result.extend_from_slice(&storage_prefix_hashed[..]);
-
-		result
+		let result = storage_prefix(Self::module_prefix(), Self::storage_prefix());
+		result.to_vec()
 	}
 
 	/// Convert an optional value retrieved from storage to the type queried.
@@ -94,16 +85,12 @@ pub trait StorageNMap<K: KeyGenerator, V: FullCodec> {
 	where
 		K: HasKeyPrefix<KP>,
 	{
-		let module_prefix_hashed = Twox128::hash(Self::module_prefix());
-		let storage_prefix_hashed = Twox128::hash(Self::storage_prefix());
+		let storage_prefix = storage_prefix(Self::module_prefix(), Self::storage_prefix());
 		let key_hashed = <K as HasKeyPrefix<KP>>::partial_key(key);
 
-		let mut final_key = Vec::with_capacity(
-			module_prefix_hashed.len() + storage_prefix_hashed.len() + key_hashed.len(),
-		);
+		let mut final_key = Vec::with_capacity(storage_prefix.len() + key_hashed.len());
 
-		final_key.extend_from_slice(&module_prefix_hashed[..]);
-		final_key.extend_from_slice(&storage_prefix_hashed[..]);
+		final_key.extend_from_slice(&storage_prefix);
 		final_key.extend_from_slice(key_hashed.as_ref());
 
 		final_key
@@ -115,16 +102,12 @@ pub trait StorageNMap<K: KeyGenerator, V: FullCodec> {
 		KG: KeyGenerator,
 		KArg: EncodeLikeTuple<KG::KArg> + TupleToEncodedIter,
 	{
-		let module_prefix_hashed = Twox128::hash(Self::module_prefix());
-		let storage_prefix_hashed = Twox128::hash(Self::storage_prefix());
+		let storage_prefix = storage_prefix(Self::module_prefix(), Self::storage_prefix());
 		let key_hashed = KG::final_key(key);
 
-		let mut final_key = Vec::with_capacity(
-			module_prefix_hashed.len() + storage_prefix_hashed.len() + key_hashed.len(),
-		);
+		let mut final_key = Vec::with_capacity(storage_prefix.len() + key_hashed.len());
 
-		final_key.extend_from_slice(&module_prefix_hashed[..]);
-		final_key.extend_from_slice(&storage_prefix_hashed[..]);
+		final_key.extend_from_slice(&storage_prefix);
 		final_key.extend_from_slice(key_hashed.as_ref());
 
 		final_key
@@ -286,16 +269,12 @@ where
 		KArg: EncodeLikeTuple<K::KArg> + TupleToEncodedIter,
 	{
 		let old_key = {
-			let module_prefix_hashed = Twox128::hash(Self::module_prefix());
-			let storage_prefix_hashed = Twox128::hash(Self::storage_prefix());
+			let storage_prefix = storage_prefix(Self::module_prefix(), Self::storage_prefix());
 			let key_hashed = K::migrate_key(&key, hash_fns);
 
-			let mut final_key = Vec::with_capacity(
-				module_prefix_hashed.len() + storage_prefix_hashed.len() + key_hashed.len(),
-			);
+			let mut final_key = Vec::with_capacity(storage_prefix.len() + key_hashed.len());
 
-			final_key.extend_from_slice(&module_prefix_hashed[..]);
-			final_key.extend_from_slice(&storage_prefix_hashed[..]);
+			final_key.extend_from_slice(&storage_prefix);
 			final_key.extend_from_slice(key_hashed.as_ref());
 
 			final_key

substrate/frame/support/src/storage/generator/value.rs

@@ -16,7 +16,6 @@
 // limitations under the License.
 
 use crate::{
-	hash::{StorageHasher, Twox128},
 	storage::{self, unhashed, StorageAppend},
 	Never,
 };
@@ -46,10 +45,7 @@ pub trait StorageValue<T: FullCodec> {
 
 	/// Generate the full key used in top storage.
 	fn storage_value_final_key() -> [u8; 32] {
-		let mut final_key = [0u8; 32];
-		final_key[0..16].copy_from_slice(&Twox128::hash(Self::module_prefix()));
-		final_key[16..32].copy_from_slice(&Twox128::hash(Self::storage_prefix()));
-		final_key
+		crate::storage::storage_prefix(Self::module_prefix(), Self::storage_prefix())
 	}
 }
 

substrate/frame/support/src/storage/migration.rs

@@ -17,7 +17,11 @@
 
 //! Some utilities for helping access storage with arbitrary key types.
 
-use crate::{hash::ReversibleStorageHasher, storage::unhashed, StorageHasher, Twox128};
+use crate::{
+	hash::ReversibleStorageHasher,
+	storage::{storage_prefix, unhashed},
+	StorageHasher, Twox128,
+};
 use codec::{Decode, Encode};
 use sp_std::prelude::*;
 
@@ -47,8 +51,8 @@ impl<T> StorageIterator<T> {
 	)]
 	pub fn with_suffix(module: &[u8], item: &[u8], suffix: &[u8]) -> Self {
 		let mut prefix = Vec::new();
-		prefix.extend_from_slice(&Twox128::hash(module));
-		prefix.extend_from_slice(&Twox128::hash(item));
+		let storage_prefix = storage_prefix(module, item);
+		prefix.extend_from_slice(&storage_prefix);
 		prefix.extend_from_slice(suffix);
 		let previous_key = prefix.clone();
 		Self { prefix, previous_key, drain: false, _phantom: Default::default() }
@@ -112,8 +116,8 @@ impl<K, T, H: ReversibleStorageHasher> StorageKeyIterator<K, T, H> {
 	)]
 	pub fn with_suffix(module: &[u8], item: &[u8], suffix: &[u8]) -> Self {
 		let mut prefix = Vec::new();
-		prefix.extend_from_slice(&Twox128::hash(module));
-		prefix.extend_from_slice(&Twox128::hash(item));
+		let storage_prefix = storage_prefix(module, item);
+		prefix.extend_from_slice(&storage_prefix);
 		prefix.extend_from_slice(suffix);
 		let previous_key = prefix.clone();
 		Self { prefix, previous_key, drain: false, _phantom: Default::default() }
@@ -173,8 +177,8 @@ pub fn storage_iter_with_suffix<T: Decode + Sized>(
 	suffix: &[u8],
 ) -> PrefixIterator<(Vec<u8>, T)> {
 	let mut prefix = Vec::new();
-	prefix.extend_from_slice(&Twox128::hash(module));
-	prefix.extend_from_slice(&Twox128::hash(item));
+	let storage_prefix = storage_prefix(module, item);
+	prefix.extend_from_slice(&storage_prefix);
 	prefix.extend_from_slice(suffix);
 	let previous_key = prefix.clone();
 	let closure = |raw_key_without_prefix: &[u8], raw_value: &[u8]| {
@@ -204,8 +208,9 @@ pub fn storage_key_iter_with_suffix<
 	suffix: &[u8],
 ) -> PrefixIterator<(K, T)> {
 	let mut prefix = Vec::new();
-	prefix.extend_from_slice(&Twox128::hash(module));
-	prefix.extend_from_slice(&Twox128::hash(item));
+	let storage_prefix = storage_prefix(module, item);
+
+	prefix.extend_from_slice(&storage_prefix);
 	prefix.extend_from_slice(suffix);
 	let previous_key = prefix.clone();
 	let closure = |raw_key_without_prefix: &[u8], raw_value: &[u8]| {
@@ -225,8 +230,8 @@ pub fn have_storage_value(module: &[u8], item: &[u8], hash: &[u8]) -> bool {
 /// Get a particular value in storage by the `module`, the map's `item` name and the key `hash`.
 pub fn get_storage_value<T: Decode + Sized>(module: &[u8], item: &[u8], hash: &[u8]) -> Option<T> {
 	let mut key = vec![0u8; 32 + hash.len()];
-	key[0..16].copy_from_slice(&Twox128::hash(module));
-	key[16..32].copy_from_slice(&Twox128::hash(item));
+	let storage_prefix = storage_prefix(module, item);
+	key[0..32].copy_from_slice(&storage_prefix);
 	key[32..].copy_from_slice(hash);
 	frame_support::storage::unhashed::get::<T>(&key)
 }
@@ -234,8 +239,8 @@ pub fn get_storage_value<T: Decode + Sized>(module: &[u8], item: &[u8], hash: &[
 /// Take a particular value in storage by the `module`, the map's `item` name and the key `hash`.
 pub fn take_storage_value<T: Decode + Sized>(module: &[u8], item: &[u8], hash: &[u8]) -> Option<T> {
 	let mut key = vec![0u8; 32 + hash.len()];
-	key[0..16].copy_from_slice(&Twox128::hash(module));
-	key[16..32].copy_from_slice(&Twox128::hash(item));
+	let storage_prefix = storage_prefix(module, item);
+	key[0..32].copy_from_slice(&storage_prefix);
 	key[32..].copy_from_slice(hash);
 	frame_support::storage::unhashed::take::<T>(&key)
 }
@@ -243,8 +248,8 @@ pub fn take_storage_value<T: Decode + Sized>(module: &[u8], item: &[u8], hash: &
 /// Put a particular value into storage by the `module`, the map's `item` name and the key `hash`.
 pub fn put_storage_value<T: Encode>(module: &[u8], item: &[u8], hash: &[u8], value: T) {
 	let mut key = vec![0u8; 32 + hash.len()];
-	key[0..16].copy_from_slice(&Twox128::hash(module));
-	key[16..32].copy_from_slice(&Twox128::hash(item));
+	let storage_prefix = storage_prefix(module, item);
+	key[0..32].copy_from_slice(&storage_prefix);
 	key[32..].copy_from_slice(hash);
 	frame_support::storage::unhashed::put(&key, &value);
 }
@@ -253,8 +258,8 @@ pub fn put_storage_value<T: Encode>(module: &[u8], item: &[u8], hash: &[u8], val
 /// `hash`.
 pub fn remove_storage_prefix(module: &[u8], item: &[u8], hash: &[u8]) {
 	let mut key = vec![0u8; 32 + hash.len()];
-	key[0..16].copy_from_slice(&Twox128::hash(module));
-	key[16..32].copy_from_slice(&Twox128::hash(item));
+	let storage_prefix = storage_prefix(module, item);
+	key[0..32].copy_from_slice(&storage_prefix);
 	key[32..].copy_from_slice(hash);
 	frame_support::storage::unhashed::kill_prefix(&key, None);
 }
@@ -293,13 +298,8 @@ pub fn move_storage_from_pallet(
 	old_pallet_name: &[u8],
 	new_pallet_name: &[u8],
 ) {
-	let mut new_prefix = Vec::new();
-	new_prefix.extend_from_slice(&Twox128::hash(new_pallet_name));
-	new_prefix.extend_from_slice(&Twox128::hash(storage_name));
-
-	let mut old_prefix = Vec::new();
-	old_prefix.extend_from_slice(&Twox128::hash(old_pallet_name));
-	old_prefix.extend_from_slice(&Twox128::hash(storage_name));
+	let new_prefix = storage_prefix(new_pallet_name, storage_name);
+	let old_prefix = storage_prefix(old_pallet_name, storage_name);
 
 	move_prefix(&old_prefix, &new_prefix);
 

substrate/frame/support/src/storage/mod.rs

@@ -18,7 +18,7 @@
 //! Stuff to do with the runtime's storage.
 
 use crate::{
-	hash::{ReversibleStorageHasher, StorageHasher, Twox128},
+	hash::{ReversibleStorageHasher, StorageHasher},
 	storage::types::{
 		EncodeLikeTuple, HasKeyPrefix, HasReversibleKeyPrefix, KeyGenerator,
 		ReversibleKeyGenerator, TupleToEncodedIter,
@@ -1108,10 +1108,7 @@ pub trait StoragePrefixedMap<Value: FullCodec> {
 
 	/// Final full prefix that prefixes all keys.
 	fn final_prefix() -> [u8; 32] {
-		let mut final_key = [0u8; 32];
-		final_key[0..16].copy_from_slice(&Twox128::hash(Self::module_prefix()));
-		final_key[16..32].copy_from_slice(&Twox128::hash(Self::storage_prefix()));
-		final_key
+		crate::storage::storage_prefix(Self::module_prefix(), Self::storage_prefix())
 	}
 
 	/// Remove all value of the storage.
@@ -1361,10 +1358,24 @@ where
 	}
 }
 
+/// Returns the storage prefix for a specific pallet name and storage name.
+///
+/// The storage prefix is `concat(twox_128(pallet_name), twox_128(storage_name))`.
+pub fn storage_prefix(pallet_name: &[u8], storage_name: &[u8]) -> [u8; 32] {
+	let pallet_hash = sp_io::hashing::twox_128(pallet_name);
+	let storage_hash = sp_io::hashing::twox_128(storage_name);
+
+	let mut final_key = [0u8; 32];
+	final_key[..16].copy_from_slice(&pallet_hash);
+	final_key[16..].copy_from_slice(&storage_hash);
+
+	final_key
+}
+
 #[cfg(test)]
 mod test {
 	use super::*;
-	use crate::{assert_ok, hash::Identity};
+	use crate::{assert_ok, hash::Identity, Twox128};
 	use bounded_vec::BoundedVec;
 	use core::convert::{TryFrom, TryInto};
 	use generator::StorageValue as _;