Refactor srml-support/storage (#3702)

* refactor

* fix

* remove unused vec storages

* address comment

substrate/srml/support/src/double_map.rs

@@ -1,140 +0,0 @@
-// Copyright 2017-2019 Parity Technologies (UK) Ltd.
-// This file is part of Substrate.
-
-// Substrate is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Substrate is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
-
-//! An implementation of double map backed by storage.
-
-use crate::rstd::prelude::*;
-use crate::codec::{Codec, Encode};
-use crate::storage::unhashed;
-use rstd::borrow::Borrow;
-
-/// An implementation of a map with a two keys.
-///
-/// It provides an important ability to efficiently remove all entries
-/// that have a common first key.
-///
-/// # Mapping of keys to a storage path
-///
-/// The storage key (i.e. the key under which the `Value` will be stored) is created from two parts.
-/// The first part is a hash of a concatenation of the `PREFIX` and `Key1`. And the second part
-/// is a hash of a `Key2`.
-///
-/// Hasher are implemented in derive_key* methods.
-pub trait StorageDoubleMapWithHasher {
-	type Key1: Encode;
-	type Key2: Encode;
-	type Value: Codec + Default;
-
-	const PREFIX: &'static [u8];
-
-	/// Insert an entry into this map.
-	fn insert<Q, R>(k1: &Q, k2: &R, val: Self::Value)
-	where
-		Self::Key1: Borrow<Q>,
-		Self::Key2: Borrow<R>,
-		Q: Codec,
-		R: Codec
-	{
-		unhashed::put(&Self::full_key(k1, k2)[..], &val);
-	}
-
-	/// Remove an entry from this map.
-	fn remove<Q, R>(k1: &Q, k2: &R)
-	where
-		Self::Key1: Borrow<Q>,
-		Self::Key2: Borrow<R>,
-		Q: Codec,
-		R: Codec
-	{
-		unhashed::kill(&Self::full_key(k1, k2)[..]);
-	}
-
-	/// Get an entry from this map.
-	///
-	/// If there is no entry stored under the given keys, returns `None`.
-	fn get<Q, R>(k1: &Q, k2: &R) -> Option<Self::Value>
-	where
-		Self::Key1: Borrow<Q>,
-		Self::Key2: Borrow<R>,
-		Q: Codec,
-		R: Codec
-	{
-		unhashed::get(&Self::full_key(k1, k2)[..])
-	}
-
-	/// Returns `true` if value under the specified keys exists.
-	fn exists<Q, R>(k1: &Q, k2: &R) -> bool
-	where
-		Self::Key1: Borrow<Q>,
-		Self::Key2: Borrow<R>,
-		Q: Codec,
-		R: Codec
-	{
-		unhashed::exists(&Self::full_key(k1, k2)[..])
-	}
-
-	/// Removes all entries that shares the `k1` as the first key.
-	fn remove_prefix<Q>(k1: &Q)
-	where
-		Self::Key1: Borrow<Q>,
-		Q: Codec
-	{
-		unhashed::kill_prefix(&Self::derive_key1(Self::encode_key1(k1)))
-	}
-
-	/// Encode key1 into Vec<u8> and prepend a prefix
-	fn encode_key1<Q>(key: &Q) -> Vec<u8>
-	where
-		Self::Key1: Borrow<Q>,
-		Q: Codec
-	{
-		let mut raw_prefix = Vec::new();
-		raw_prefix.extend(Self::PREFIX);
-		key.encode_to(&mut raw_prefix);
-		raw_prefix
-	}
-
-	/// Encode key2 into Vec<u8>
-	fn encode_key2<R>(key: &R) -> Vec<u8>
-	where
-		Self::Key2: Borrow<R>,
-		R: Codec
-	{
-		Encode::encode(&key)
-	}
-
-	/// Derive the first part of the key
-	fn derive_key1(key1_data: Vec<u8>) -> Vec<u8>;
-
-	/// Derive the remaining part of the key
-	fn derive_key2(key2_data: Vec<u8>) -> Vec<u8>;
-
-	/// Returns a compound key that consist of the two parts: (prefix, `k1`) and `k2`.
-	/// The first part is hashed and then concatenated with a hash of `k2`.
-	fn full_key<Q, R>(k1: &Q, k2: &R) -> Vec<u8>
-	where
-		Self::Key1: Borrow<Q>,
-		Self::Key2: Borrow<R>,
-		Q: Codec,
-		R: Codec
-	{
-		let key1_data = Self::encode_key1(k1);
-		let key2_data = Self::encode_key2(k2);
-		let mut key = Self::derive_key1(key1_data);
-		key.extend(Self::derive_key2(key2_data));
-		key
-	}
-}

substrate/srml/support/src/hash.rs

@@ -0,0 +1,114 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Hash utilities.
+
+use codec::Codec;
+use rstd::prelude::Vec;
+use runtime_io::{blake2_128, blake2_256, twox_64, twox_128, twox_256};
+
+// This trait must be kept coherent with srml-support-procedural HasherKind usage
+pub trait Hashable: Sized {
+	fn blake2_128(&self) -> [u8; 16];
+	fn blake2_256(&self) -> [u8; 32];
+	fn twox_128(&self) -> [u8; 16];
+	fn twox_256(&self) -> [u8; 32];
+	fn twox_64_concat(&self) -> Vec<u8>;
+}
+
+impl<T: Codec> Hashable for T {
+	fn blake2_128(&self) -> [u8; 16] {
+		self.using_encoded(blake2_128)
+	}
+	fn blake2_256(&self) -> [u8; 32] {
+		self.using_encoded(blake2_256)
+	}
+	fn twox_128(&self) -> [u8; 16] {
+		self.using_encoded(twox_128)
+	}
+	fn twox_256(&self) -> [u8; 32] {
+		self.using_encoded(twox_256)
+	}
+	fn twox_64_concat(&self) -> Vec<u8> {
+		self.using_encoded(Twox64Concat::hash)
+	}
+}
+
+/// Hasher to use to hash keys to insert to storage.
+pub trait StorageHasher: 'static {
+	type Output: AsRef<[u8]>;
+	fn hash(x: &[u8]) -> Self::Output;
+}
+
+/// Hash storage keys with `concat(twox64(key), key)`
+pub struct Twox64Concat;
+impl StorageHasher for Twox64Concat {
+	type Output = Vec<u8>;
+	fn hash(x: &[u8]) -> Vec<u8> {
+		twox_64(x)
+			.into_iter()
+			.chain(x.into_iter())
+			.cloned()
+			.collect::<Vec<_>>()
+	}
+}
+
+/// Hash storage keys with blake2 128
+pub struct Blake2_128;
+impl StorageHasher for Blake2_128 {
+	type Output = [u8; 16];
+	fn hash(x: &[u8]) -> [u8; 16] {
+		blake2_128(x)
+	}
+}
+
+/// Hash storage keys with blake2 256
+pub struct Blake2_256;
+impl StorageHasher for Blake2_256 {
+	type Output = [u8; 32];
+	fn hash(x: &[u8]) -> [u8; 32] {
+		blake2_256(x)
+	}
+}
+
+/// Hash storage keys with twox 128
+pub struct Twox128;
+impl StorageHasher for Twox128 {
+	type Output = [u8; 16];
+	fn hash(x: &[u8]) -> [u8; 16] {
+		twox_128(x)
+	}
+}
+
+/// Hash storage keys with twox 256
+pub struct Twox256;
+impl StorageHasher for Twox256 {
+	type Output = [u8; 32];
+	fn hash(x: &[u8]) -> [u8; 32] {
+		twox_256(x)
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn test_twox_64_concat() {
+		let r = Twox64Concat::hash(b"foo");
+		assert_eq!(r.split_at(8), (&twox_128(b"foo")[..8], &b"foo"[..]))
+	}
+}

substrate/srml/support/src/hashable.rs

@@ -1,50 +0,0 @@
-// Copyright 2017-2019 Parity Technologies (UK) Ltd.
-// This file is part of Substrate.
-
-// Substrate is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Substrate is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
-
-//! Hashable trait.
-
-use crate::codec::Codec;
-use runtime_io::{blake2_128, blake2_256, twox_128, twox_256};
-use crate::storage::hashed::StorageHasher;
-use crate::Twox64Concat;
-use crate::rstd::prelude::Vec;
-
-// This trait must be kept coherent with srml-support-procedural HasherKind usage
-pub trait Hashable: Sized {
-	fn blake2_128(&self) -> [u8; 16];
-	fn blake2_256(&self) -> [u8; 32];
-	fn twox_128(&self) -> [u8; 16];
-	fn twox_256(&self) -> [u8; 32];
-	fn twox_64_concat(&self) -> Vec<u8>;
-}
-
-impl<T: Codec> Hashable for T {
-	fn blake2_128(&self) -> [u8; 16] {
-		self.using_encoded(blake2_128)
-	}
-	fn blake2_256(&self) -> [u8; 32] {
-		self.using_encoded(blake2_256)
-	}
-	fn twox_128(&self) -> [u8; 16] {
-		self.using_encoded(twox_128)
-	}
-	fn twox_256(&self) -> [u8; 32] {
-		self.using_encoded(twox_256)
-	}
-	fn twox_64_concat(&self) -> Vec<u8> {
-		self.using_encoded(Twox64Concat::hash)
-	}
-}

substrate/srml/support/src/lib.rs

@@ -38,14 +38,14 @@ pub use paste;
 #[cfg(feature = "std")]
 #[doc(hidden)]
 pub use runtime_io::with_storage;
-
-pub use self::storage::hashed::{Twox256, Twox128, Blake2_256, Blake2_128, Twox64Concat};
+#[doc(hidden)]
+pub use runtime_io::storage_root;
 
 #[macro_use]
 pub mod dispatch;
 #[macro_use]
 pub mod storage;
-mod hashable;
+mod hash;
 #[macro_use]
 pub mod event;
 #[macro_use]
@@ -60,14 +60,11 @@ pub mod inherent;
 pub mod unsigned;
 #[macro_use]
 pub mod error;
-mod double_map;
 pub mod traits;
 
+pub use self::hash::{Twox256, Twox128, Blake2_256, Blake2_128, Twox64Concat, Hashable};
 pub use self::storage::{StorageValue, StorageMap, StorageLinkedMap, StorageDoubleMap};
-pub use self::hashable::Hashable;
 pub use self::dispatch::{Parameter, Callable, IsSubType};
-pub use self::double_map::StorageDoubleMapWithHasher;
-pub use runtime_io::storage_root;
 pub use sr_primitives::{self, ConsensusEngineId, print, traits::Printable};
 
 /// Macro for easily creating a new implementation of the `Get` trait. Use similarly to

substrate/srml/support/src/storage/generator/double_map.rs

@@ -15,8 +15,9 @@
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
 use rstd::prelude::*;
+use rstd::borrow::Borrow;
 use codec::{Codec, Encode, EncodeAppend};
-use crate::{storage::{self, unhashed, hashed::StorageHasher}, rstd::borrow::Borrow};
+use crate::{storage::{self, unhashed}, hash::StorageHasher};
 
 /// Generator for `StorageDoubleMap` used by `decl_storage`.
 ///

substrate/srml/support/src/storage/generator/linked_map.rs

@@ -15,7 +15,7 @@
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
 use codec::{Codec, Encode, Decode};
-use crate::{storage::{self, unhashed, hashed::StorageHasher}, traits::Len};
+use crate::{storage::{self, unhashed}, hash::StorageHasher, traits::Len};
 use rstd::{
 	borrow::Borrow,
 	marker::PhantomData,

substrate/srml/support/src/storage/generator/map.rs

@@ -18,7 +18,7 @@
 use rstd::prelude::*;
 use rstd::borrow::Borrow;
 use codec::{Codec, Encode};
-use crate::{storage::{self, unhashed, hashed::StorageHasher}, traits::Len};
+use crate::{storage::{self, unhashed}, hash::StorageHasher, traits::Len};
 
 /// Generator for `StorageMap` used by `decl_storage`.
 ///

substrate/srml/support/src/storage/generator/value.rs

@@ -18,7 +18,7 @@
 use rstd::prelude::*;
 use rstd::{borrow::Borrow, iter::FromIterator};
 use codec::{Codec, Encode};
-use crate::{storage::{self, unhashed, hashed::{Twox128, StorageHasher}}, traits::Len};
+use crate::{storage::{self, unhashed}, hash::{Twox128, StorageHasher}, traits::Len};
 
 /// Generator for `StorageValue` used by `decl_storage`.
 ///

substrate/srml/support/src/storage/hashed.rs

@@ -17,71 +17,8 @@
 //! Operation on runtime storage using hashed keys.
 
 use super::unhashed;
-use crate::rstd::prelude::*;
-use crate::rstd::borrow::Borrow;
-use crate::codec::{Codec, Encode, Decode, KeyedVec};
-use runtime_io::{self, twox_64, twox_128, blake2_128, twox_256, blake2_256};
-
-/// Hasher to use to hash keys to insert to storage.
-pub trait StorageHasher: 'static {
-	type Output: AsRef<[u8]>;
-	fn hash(x: &[u8]) -> Self::Output;
-}
-
-/// Hash storage keys with `concat(twox64(key), key)`
-pub struct Twox64Concat;
-impl StorageHasher for Twox64Concat {
-	type Output = Vec<u8>;
-	fn hash(x: &[u8]) -> Vec<u8> {
-		twox_64(x)
-			.into_iter()
-			.chain(x.into_iter())
-			.cloned()
-			.collect::<Vec<_>>()
-	}
-}
-
-#[test]
-fn test_twox_64_concat() {
-	let r = Twox64Concat::hash(b"foo");
-	assert_eq!(r.split_at(8), (&twox_128(b"foo")[..8], &b"foo"[..]))
-}
-
-/// Hash storage keys with blake2 128
-pub struct Blake2_128;
-impl StorageHasher for Blake2_128 {
-	type Output = [u8; 16];
-	fn hash(x: &[u8]) -> [u8; 16] {
-		blake2_128(x)
-	}
-}
-
-/// Hash storage keys with blake2 256
-pub struct Blake2_256;
-impl StorageHasher for Blake2_256 {
-	type Output = [u8; 32];
-	fn hash(x: &[u8]) -> [u8; 32] {
-		blake2_256(x)
-	}
-}
-
-/// Hash storage keys with twox 128
-pub struct Twox128;
-impl StorageHasher for Twox128 {
-	type Output = [u8; 16];
-	fn hash(x: &[u8]) -> [u8; 16] {
-		twox_128(x)
-	}
-}
-
-/// Hash storage keys with twox 256
-pub struct Twox256;
-impl StorageHasher for Twox256 {
-	type Output = [u8; 32];
-	fn hash(x: &[u8]) -> [u8; 32] {
-		twox_256(x)
-	}
-}
+use rstd::prelude::*;
+use codec::{Encode, Decode};
 
 /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry.
 pub fn get<T, HashFn, R>(hash: &HashFn, key: &[u8]) -> Option<T>
@@ -216,131 +153,3 @@ where
 {
 	unhashed::put_raw(&hash(key).as_ref(), value)
 }
-
-/// A trait to conveniently store a vector of storable data.
-///
-/// It uses twox_128 hasher. Final keys in trie are `twox_128(concatenation(PREFIX,count))`
-pub trait StorageVec {
-	type Item: Default + Sized + Codec;
-	const PREFIX: &'static [u8];
-
-	/// Get the current set of items.
-	fn items() -> Vec<Self::Item> {
-		(0..Self::count()).into_iter().map(Self::item).collect()
-	}
-
-	/// Set the current set of items.
-	fn set_items<I, T>(items: I)
-		where
-			I: IntoIterator<Item=T>,
-			T: Borrow<Self::Item>,
-	{
-		let mut count: u32 = 0;
-
-		for i in items.into_iter() {
-			put(&twox_128, &count.to_keyed_vec(Self::PREFIX), i.borrow());
-			count = count.checked_add(1).expect("exceeded runtime storage capacity");
-		}
-
-		Self::set_count(count);
-	}
-
-	/// Push an item.
-	fn push(item: &Self::Item) {
-		let len = Self::count();
-		put(&twox_128, &len.to_keyed_vec(Self::PREFIX), item);
-		Self::set_count(len + 1);
-	}
-
-	fn set_item(index: u32, item: &Self::Item) {
-		if index < Self::count() {
-			put(&twox_128, &index.to_keyed_vec(Self::PREFIX), item);
-		}
-	}
-
-	fn clear_item(index: u32) {
-		if index < Self::count() {
-			kill(&twox_128, &index.to_keyed_vec(Self::PREFIX));
-		}
-	}
-
-	fn item(index: u32) -> Self::Item {
-		get_or_default(&twox_128, &index.to_keyed_vec(Self::PREFIX))
-	}
-
-	fn set_count(count: u32) {
-		(count..Self::count()).for_each(Self::clear_item);
-		put(&twox_128, &b"len".to_keyed_vec(Self::PREFIX), &count);
-	}
-
-	fn count() -> u32 {
-		get_or_default(&twox_128, &b"len".to_keyed_vec(Self::PREFIX))
-	}
-}
-
-#[cfg(test)]
-mod tests {
-	use super::*;
-	use runtime_io::{twox_128, TestExternalities, with_externalities};
-
-	#[test]
-	fn integers_can_be_stored() {
-		let mut t = TestExternalities::default();
-		with_externalities(&mut t, || {
-			let x = 69u32;
-			put(&twox_128, b":test", &x);
-			let y: u32 = get(&twox_128, b":test").unwrap();
-			assert_eq!(x, y);
-		});
-		with_externalities(&mut t, || {
-			let x = 69426942i64;
-			put(&twox_128, b":test", &x);
-			let y: i64 = get(&twox_128, b":test").unwrap();
-			assert_eq!(x, y);
-		});
-	}
-
-	#[test]
-	fn bools_can_be_stored() {
-		let mut t = TestExternalities::default();
-		with_externalities(&mut t, || {
-			let x = true;
-			put(&twox_128, b":test", &x);
-			let y: bool = get(&twox_128, b":test").unwrap();
-			assert_eq!(x, y);
-		});
-
-		with_externalities(&mut t, || {
-			let x = false;
-			put(&twox_128, b":test", &x);
-			let y: bool = get(&twox_128, b":test").unwrap();
-			assert_eq!(x, y);
-		});
-	}
-
-	#[test]
-	fn vecs_can_be_retrieved() {
-		let mut t = TestExternalities::default();
-		with_externalities(&mut t, || {
-			runtime_io::set_storage(&twox_128(b":test"), b"\x2cHello world");
-			let x = b"Hello world".to_vec();
-			let y = get::<Vec<u8>, _, _>(&twox_128, b":test").unwrap();
-			assert_eq!(x, y);
-		});
-	}
-
-	#[test]
-	fn vecs_can_be_stored() {
-		let mut t = TestExternalities::default();
-		let x = b"Hello world".to_vec();
-
-		with_externalities(&mut t, || {
-			put(&twox_128, b":test", &x);
-		});
-
-		with_externalities(&mut t, || {
-			let y: Vec<u8> = get(&twox_128, b":test").unwrap();
-			assert_eq!(x, y);
-		});
-	}
-}

substrate/srml/support/src/storage/mod.rs

@@ -18,7 +18,7 @@
 
 use crate::rstd::prelude::*;
 use crate::rstd::{borrow::Borrow, iter::FromIterator};
-use codec::{Codec, Encode, Decode, KeyedVec, EncodeAppend};
+use codec::{Codec, Encode, Decode, EncodeAppend};
 use crate::traits::Len;
 
 #[macro_use]

substrate/srml/support/src/storage/unhashed.rs

@@ -16,8 +16,7 @@
 
 //! Operation on unhashed runtime storage.
 
-use crate::rstd::borrow::Borrow;
-use super::{Codec, Encode, Decode, KeyedVec, Vec};
+use super::{Encode, Decode, Vec};
 
 /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry.
 pub fn get<T: Decode + Sized>(key: &[u8]) -> Option<T> {
@@ -100,55 +99,3 @@ pub fn get_raw(key: &[u8]) -> Option<Vec<u8>> {
 pub fn put_raw(key: &[u8], value: &[u8]) {
 	runtime_io::set_storage(key, value)
 }
-
-/// A trait to conveniently store a vector of storable data.
-pub trait StorageVec {
-	type Item: Default + Sized + Codec;
-	const PREFIX: &'static [u8];
-
-	/// Get the current set of items.
-	fn items() -> Vec<Self::Item> {
-		(0..Self::count()).into_iter().map(Self::item).collect()
-	}
-
-	/// Set the current set of items.
-	fn set_items<I, T>(items: I)
-		where
-			I: IntoIterator<Item=T>,
-			T: Borrow<Self::Item>,
-	{
-		let mut count: u32 = 0;
-
-		for i in items.into_iter() {
-			put(&count.to_keyed_vec(Self::PREFIX), i.borrow());
-			count = count.checked_add(1).expect("exceeded runtime storage capacity");
-		}
-
-		Self::set_count(count);
-	}
-
-	fn set_item(index: u32, item: &Self::Item) {
-		if index < Self::count() {
-			put(&index.to_keyed_vec(Self::PREFIX), item);
-		}
-	}
-
-	fn clear_item(index: u32) {
-		if index < Self::count() {
-			kill(&index.to_keyed_vec(Self::PREFIX));
-		}
-	}
-
-	fn item(index: u32) -> Self::Item {
-		get_or_default(&index.to_keyed_vec(Self::PREFIX))
-	}
-
-	fn set_count(count: u32) {
-		(count..Self::count()).for_each(Self::clear_item);
-		put(&b"len".to_keyed_vec(Self::PREFIX), &count);
-	}
-
-	fn count() -> u32 {
-		get_or_default(&b"len".to_keyed_vec(Self::PREFIX))
-	}
-}

substrate/srml/support/src/traits.rs

@@ -18,11 +18,13 @@
 //!
 //! NOTE: If you're looking for `parameter_types`, it has moved in to the top-level module.
 
-use crate::rstd::{prelude::*, result, marker::PhantomData, ops::Div};
-use crate::codec::{Codec, Encode, Decode};
+use rstd::{prelude::*, result, marker::PhantomData, ops::Div};
+use codec::{Codec, Encode, Decode};
 use primitives::u32_trait::Value as U32;
-use crate::sr_primitives::traits::{MaybeSerializeDebug, SimpleArithmetic, Saturating};
-use crate::sr_primitives::ConsensusEngineId;
+use sr_primitives::{
+	ConsensusEngineId,
+	traits::{MaybeSerializeDebug, SimpleArithmetic, Saturating},
+};
 
 /// Anything that can have a `::len()` method.
 pub trait Len {