Implement all storage after prefix (#4227)

* Implement all storage after prefix

* fix test, bump version and fix doc

* bump metadata version

* Update frame/support/procedural/src/storage/storage_struct.rs
This commit is contained in:
thiolliere
2019-11-27 19:23:20 +01:00
committed by Gavin Wood
parent dcaabbaacf
commit f4035cd5ac
15 changed files with 310 additions and 325 deletions
@@ -17,7 +17,7 @@
use rstd::prelude::*;
use rstd::borrow::Borrow;
use codec::{Ref, FullCodec, FullEncode, Encode, EncodeLike, EncodeAppend};
use crate::{storage::{self, unhashed}, hash::StorageHasher};
use crate::{storage::{self, unhashed}, hash::{StorageHasher, Twox128}};
/// Generator for `StorageDoubleMap` used by `decl_storage`.
///
@@ -29,7 +29,7 @@ use crate::{storage::{self, unhashed}, hash::StorageHasher};
///
/// Thus value for (key1, key2) is stored at:
/// ```nocompile
/// Hasher1(key1_prefix ++ key1) ++ Hasher2(key2)
/// Twox128(module_prefix) ++ Twox128(storage_prefix) ++ Hasher1(encode(key1)) ++ Hasher2(encode(key2))
/// ```
///
/// # Warning
@@ -49,8 +49,11 @@ pub trait StorageDoubleMap<K1: FullEncode, K2: FullEncode, V: FullCodec> {
/// Hasher for the second key.
type Hasher2: StorageHasher;
/// Get the prefix for first key.
fn key1_prefix() -> &'static [u8];
/// Module prefix. Used for generating final key.
fn module_prefix() -> &'static [u8];
/// Storage prefix. Used for generating final key.
fn storage_prefix() -> &'static [u8];
/// Convert an optional value retrieved from storage to the type queried.
fn from_optional_value_to_query(v: Option<V>) -> Self::Query;
@@ -59,13 +62,23 @@ pub trait StorageDoubleMap<K1: FullEncode, K2: FullEncode, V: FullCodec> {
fn from_query_to_optional_value(v: Self::Query) -> Option<V>;
/// Generate the first part of the key used in top storage.
fn storage_double_map_final_key1<KArg1>(k1: KArg1) -> <Self::Hasher1 as StorageHasher>::Output
fn storage_double_map_final_key1<KArg1>(k1: KArg1) -> Vec<u8>
where
KArg1: EncodeLike<K1>,
{
let mut final_key1 = Self::key1_prefix().to_vec();
k1.encode_to(&mut final_key1);
Self::Hasher1::hash(&final_key1)
let module_prefix_hashed = Twox128::hash(Self::module_prefix());
let storage_prefix_hashed = Twox128::hash(Self::storage_prefix());
let key_hashed = k1.borrow().using_encoded(Self::Hasher1::hash);
let mut final_key = Vec::with_capacity(
module_prefix_hashed.len() + storage_prefix_hashed.len() + key_hashed.as_ref().len()
);
final_key.extend_from_slice(&module_prefix_hashed[..]);
final_key.extend_from_slice(&storage_prefix_hashed[..]);
final_key.extend_from_slice(key_hashed.as_ref());
final_key
}
/// Generate the full key used in top storage.
@@ -74,7 +87,7 @@ pub trait StorageDoubleMap<K1: FullEncode, K2: FullEncode, V: FullCodec> {
KArg1: EncodeLike<K1>,
KArg2: EncodeLike<K2>,
{
let mut final_key = Self::storage_double_map_final_key1(k1).as_ref().to_vec();
let mut final_key = Self::storage_double_map_final_key1(k1);
final_key.extend_from_slice(k2.using_encoded(Self::Hasher2::hash).as_ref());
final_key
}
@@ -15,22 +15,52 @@
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
use codec::{FullCodec, Encode, Decode, EncodeLike, Ref};
use crate::{storage::{self, unhashed}, hash::StorageHasher, traits::Len};
use rstd::marker::PhantomData;
use crate::{storage::{self, unhashed}, hash::{StorageHasher, Twox128}, traits::Len};
use rstd::{prelude::*, marker::PhantomData};
/// Generator for `StorageLinkedMap` used by `decl_storage`.
///
/// # Mapping of keys to a storage path
/// By default final key generation rely on `KeyFormat`.
pub trait StorageLinkedMap<K: FullCodec, V: FullCodec> {
/// The type that get/take returns.
type Query;
/// The family of key formats used for this map.
type KeyFormat: KeyFormat;
/// Convert an optional value retrieved from storage to the type queried.
fn from_optional_value_to_query(v: Option<V>) -> Self::Query;
/// Convert a query to an optional value into storage.
fn from_query_to_optional_value(v: Self::Query) -> Option<V>;
/// Generate the full key used in top storage.
fn storage_linked_map_final_key<KeyArg>(key: KeyArg) -> Vec<u8>
where
KeyArg: EncodeLike<K>,
{
<Self::KeyFormat as KeyFormat>::storage_linked_map_final_key::<KeyArg>(&key)
}
/// Generate the hashed key for head
fn storage_linked_map_final_head_key() -> Vec<u8> {
<Self::KeyFormat as KeyFormat>::storage_linked_map_final_head_key()
}
}
/// A type-abstracted key format used for a family of linked-map types.
///
/// # Default mapping of keys to a storage path
///
/// The key for the head of the map is stored at one fixed path:
/// ```nocompile
/// Hasher(head_key)
/// Twox128(module_prefix) ++ Twox128(head_prefix)
/// ```
///
/// For each key, the value stored under that key is appended with a
/// [`Linkage`](struct.Linkage.html) (which hold previous and next key) at the path:
/// ```nocompile
/// Hasher(prefix ++ key)
/// Twox128(module_prefix) ++ Twox128(storage_prefix) ++ Hasher(encode(key))
/// ```
///
/// Enumeration is done by getting the head of the linked map and then iterating getting the
@@ -40,66 +70,45 @@ use rstd::marker::PhantomData;
///
/// If the keys are not trusted (e.g. can be set by a user), a cryptographic `hasher` such as
/// `blake2_256` must be used. Otherwise, other values in storage can be compromised.
pub trait StorageLinkedMap<K: FullCodec, V: FullCodec> {
/// The type that get/take returns.
type Query;
/// Hasher used to insert into storage.
type Hasher: StorageHasher;
/// The family of key formats used for this map.
type KeyFormat: KeyFormat<Hasher=Self::Hasher>;
/// Prefix used to prepend each key.
fn prefix() -> &'static [u8];
/// The head key of the linked-map.
fn head_key() -> &'static [u8] {
<Self::KeyFormat as KeyFormat>::head_key()
}
/// Convert an optional value retrieved from storage to the type queried.
fn from_optional_value_to_query(v: Option<V>) -> Self::Query;
/// Convert a query to an optional value into storage.
fn from_query_to_optional_value(v: Self::Query) -> Option<V>;
/// Generate the full key used in top storage.
fn storage_linked_map_final_key<KeyArg>(key: KeyArg) -> <Self::Hasher as StorageHasher>::Output
where
KeyArg: EncodeLike<K>,
{
<Self::KeyFormat as KeyFormat>::storage_linked_map_final_key::<KeyArg>(Self::prefix(), &key)
}
/// Generate the hashed key for head
fn storage_linked_map_final_head_key() -> <Self::Hasher as StorageHasher>::Output {
<Self::KeyFormat as KeyFormat>::storage_linked_map_final_head_key()
}
}
/// A type-abstracted key format used for a family of linked-map types.
pub trait KeyFormat {
/// Hasher. Used for generating final key and final head key.
type Hasher: StorageHasher;
/// Key used to store linked map head.
fn head_key() -> &'static [u8];
/// Module prefix. Used for generating final key.
fn module_prefix() -> &'static [u8];
/// Storage prefix. Used for generating final key.
fn storage_prefix() -> &'static [u8];
/// Storage prefix. Used for generating final head key.
fn head_prefix() -> &'static [u8];
/// Generate the full key used in top storage.
fn storage_linked_map_final_key<K>(prefix: &[u8], key: &K)
-> <Self::Hasher as StorageHasher>::Output
fn storage_linked_map_final_key<K>(key: &K) -> Vec<u8>
where
K: Encode,
{
let mut final_key = prefix.to_vec();
key.encode_to(&mut final_key);
<Self::Hasher as StorageHasher>::hash(&final_key)
let module_prefix_hashed = Twox128::hash(Self::module_prefix());
let storage_prefix_hashed = Twox128::hash(Self::storage_prefix());
let key_hashed = key.using_encoded(Self::Hasher::hash);
let mut final_key = Vec::with_capacity(
module_prefix_hashed.len() + storage_prefix_hashed.len() + key_hashed.as_ref().len()
);
final_key.extend_from_slice(&module_prefix_hashed[..]);
final_key.extend_from_slice(&storage_prefix_hashed[..]);
final_key.extend_from_slice(key_hashed.as_ref());
final_key
}
fn storage_linked_map_final_head_key()
-> <Self::Hasher as StorageHasher>::Output
{
<Self::Hasher as StorageHasher>::hash(Self::head_key())
/// Generate the full key used in top storage to store the head of the linked map.
fn storage_linked_map_final_head_key() -> Vec<u8> {
[
Twox128::hash(Self::module_prefix()),
Twox128::hash(Self::head_prefix()),
].concat()
}
}
@@ -135,17 +144,15 @@ struct EncodeLikeLinkage<PKey: EncodeLike<Key>, NKey: EncodeLike<Key>, Key: Enco
/// A key-value pair iterator for enumerable map.
pub struct Enumerator<K, V, F> {
next: Option<K>,
prefix: &'static [u8],
_phantom: PhantomData<(V, F)>,
}
impl<K, V, F> Enumerator<K, V, F> {
/// Create an explicit enumerator for testing.
#[cfg(test)]
pub fn from_head(head: K, prefix: &'static [u8]) -> Self {
pub fn from_head(head: K) -> Self {
Enumerator {
next: Some(head),
prefix,
_phantom: Default::default(),
}
}
@@ -163,15 +170,15 @@ where
let next = self.next.take()?;
let (val, linkage): (V, Linkage<K>) = {
let next_full_key = F::storage_linked_map_final_key(self.prefix, &next);
let next_full_key = F::storage_linked_map_final_key(&next);
match read_with_linkage::<K, V>(next_full_key.as_ref()) {
Some(value) => value,
None => {
// TODO #3700: error should be handleable.
runtime_print!(
"ERROR: Corrupted state: linked map head_key={:?}: \
"ERROR: Corrupted state: linked map {:?}{:?}: \
next value doesn't exist at {:?}",
F::head_key(), next_full_key.as_ref(),
F::module_prefix(), F::storage_prefix(), next_full_key,
);
return None
}
@@ -187,18 +194,14 @@ where
///
/// Takes care of updating previous and next elements points
/// as well as updates head if the element is first or last.
fn remove_linkage<K, V, F>(linkage: Linkage<K>, prefix: &[u8])
fn remove_linkage<K, V, F>(linkage: Linkage<K>)
where
K: FullCodec,
V: FullCodec,
F: KeyFormat,
{
let next_key = linkage.next.as_ref()
.map(|k| F::storage_linked_map_final_key(prefix, k))
.map(|x| x.as_ref().to_vec());
let prev_key = linkage.previous.as_ref()
.map(|k| F::storage_linked_map_final_key(prefix, k))
.map(|x| x.as_ref().to_vec());
let next_key = linkage.next.as_ref().map(|k| F::storage_linked_map_final_key(k));
let prev_key = linkage.previous.as_ref().map(|k| F::storage_linked_map_final_key(k));
if let Some(prev_key) = prev_key {
// Retrieve previous element and update `next`
@@ -208,9 +211,9 @@ where
} else {
// TODO #3700: error should be handleable.
runtime_print!(
"ERROR: Corrupted state: linked map head_key={:?}: \
"ERROR: Corrupted state: linked map {:?}{:?}: \
previous value doesn't exist at {:?}",
F::head_key(), prev_key,
F::module_prefix(), F::storage_prefix(), prev_key,
);
}
} else {
@@ -225,9 +228,9 @@ where
} else {
// TODO #3700: error should be handleable.
runtime_print!(
"ERROR: Corrupted state: linked map head_key={:?}: \
"ERROR: Corrupted state: linked map {:?}{:?}: \
next value doesn't exist at {:?}",
F::head_key(), next_key,
F::module_prefix(), F::storage_prefix(), next_key,
);
}
}
@@ -245,7 +248,7 @@ where
/// Generate linkage for newly inserted element.
///
/// Takes care of updating head and previous head's pointer.
pub(super) fn new_head_linkage<KeyArg, K, V, F>(key: KeyArg, prefix: &[u8]) -> Linkage<K>
pub(super) fn new_head_linkage<KeyArg, K, V, F>(key: KeyArg) -> Linkage<K>
where
KeyArg: EncodeLike<K>,
K: FullCodec,
@@ -255,7 +258,7 @@ where
if let Some(head) = read_head::<K, F>() {
// update previous head predecessor
{
let head_key = F::storage_linked_map_final_key(prefix, &head);
let head_key = F::storage_linked_map_final_key(&head);
if let Some((data, linkage)) = read_with_linkage::<K, V>(head_key.as_ref()) {
let new_linkage = EncodeLikeLinkage::<_, _, K> {
previous: Some(Ref::from(&key)),
@@ -266,9 +269,9 @@ where
} else {
// TODO #3700: error should be handleable.
runtime_print!(
"ERROR: Corrupted state: linked map head_key={:?}: \
"ERROR: Corrupted state: linked map {:?}{:?}: \
head value doesn't exist at {:?}",
F::head_key(), head_key.as_ref(),
F::module_prefix(), F::storage_prefix(), head_key,
);
// Thus we consider we are first - update the head and produce empty linkage
@@ -333,7 +336,6 @@ where
}
fn swap<KeyArg1: EncodeLike<K>, KeyArg2: EncodeLike<K>>(key1: KeyArg1, key2: KeyArg2) {
let prefix = Self::prefix();
let final_key1 = Self::storage_linked_map_final_key(Ref::from(&key1));
let final_key2 = Self::storage_linked_map_final_key(Ref::from(&key2));
let full_value_1 = read_with_linkage::<K, V>(final_key1.as_ref());
@@ -348,13 +350,13 @@ where
// Remove key and insert the new one.
(Some((value, _linkage)), None) => {
Self::remove(key1);
let linkage = new_head_linkage::<_, _, V, G::KeyFormat>(key2, prefix);
let linkage = new_head_linkage::<_, _, V, G::KeyFormat>(key2);
unhashed::put(final_key2.as_ref(), &(value, linkage));
}
// Remove key and insert the new one.
(None, Some((value, _linkage))) => {
Self::remove(key2);
let linkage = new_head_linkage::<_, _, V, G::KeyFormat>(key1, prefix);
let linkage = new_head_linkage::<_, _, V, G::KeyFormat>(key1);
unhashed::put(final_key1.as_ref(), &(value, linkage));
}
// No-op.
@@ -368,7 +370,7 @@ where
// overwrite but reuse existing linkage
Some((_data, linkage)) => linkage,
// create new linkage
None => new_head_linkage::<_, _, V, G::KeyFormat>(key, Self::prefix()),
None => new_head_linkage::<_, _, V, G::KeyFormat>(key),
};
unhashed::put(final_key.as_ref(), &(val, linkage))
}
@@ -398,7 +400,7 @@ where
let full_value: Option<(V, Linkage<K>)> = unhashed::take(final_key.as_ref());
let value = full_value.map(|(data, linkage)| {
remove_linkage::<K, V, G::KeyFormat>(linkage, Self::prefix());
remove_linkage::<K, V, G::KeyFormat>(linkage);
data
});
@@ -408,7 +410,6 @@ where
fn enumerate() -> Self::Enumerator {
Enumerator::<_, _, G::KeyFormat> {
next: read_head::<_, G::KeyFormat>(),
prefix: Self::prefix(),
_phantom: Default::default(),
}
}
@@ -436,7 +437,6 @@ where
where K2: FullCodec + Clone, V2: Decode, TK: Fn(K2) -> K, TV: Fn(V2) -> V
{
let head_key = read_head::<K2, G::KeyFormat>().ok_or(None)?;
let prefix = G::prefix();
let mut last_key = None;
let mut current_key = head_key.clone();
@@ -451,7 +451,7 @@ where
};
loop {
let old_raw_key = G::KeyFormat::storage_linked_map_final_key(prefix, &current_key);
let old_raw_key = G::KeyFormat::storage_linked_map_final_key(&current_key);
let x = unhashed::take(old_raw_key.as_ref());
let (val, linkage): (V2, Linkage<K2>) = match x {
Some(v) => v,
@@ -18,13 +18,13 @@
use rstd::prelude::*;
use rstd::borrow::Borrow;
use codec::{FullCodec, FullEncode, Encode, EncodeLike, Ref, EncodeAppend};
use crate::{storage::{self, unhashed}, hash::StorageHasher, traits::Len};
use crate::{storage::{self, unhashed}, hash::{StorageHasher, Twox128}, traits::Len};
/// Generator for `StorageMap` used by `decl_storage`.
///
/// For each key value is stored at:
/// By default each key value is stored at:
/// ```nocompile
/// Hasher(prefix ++ key)
/// Twox128(module_prefix) ++ Twox128(storage_prefix) ++ Hasher(encode(key))
/// ```
///
/// # Warning
@@ -35,11 +35,14 @@ pub trait StorageMap<K: FullEncode, V: FullCodec> {
/// The type that get/take returns.
type Query;
/// Hasher used to insert into storage.
/// Hasher. Used for generating final key.
type Hasher: StorageHasher;
/// Prefix used to prepend each key.
fn prefix() -> &'static [u8];
/// Module prefix. Used for generating final key.
fn module_prefix() -> &'static [u8];
/// Storage prefix. Used for generating final key.
fn storage_prefix() -> &'static [u8];
/// Convert an optional value retrieved from storage to the type queried.
fn from_optional_value_to_query(v: Option<V>) -> Self::Query;
@@ -48,13 +51,23 @@ pub trait StorageMap<K: FullEncode, V: FullCodec> {
fn from_query_to_optional_value(v: Self::Query) -> Option<V>;
/// Generate the full key used in top storage.
fn storage_map_final_key<KeyArg>(key: KeyArg) -> <Self::Hasher as StorageHasher>::Output
fn storage_map_final_key<KeyArg>(key: KeyArg) -> Vec<u8>
where
KeyArg: EncodeLike<K>,
{
let mut final_key = Self::prefix().to_vec();
key.borrow().encode_to(&mut final_key);
Self::Hasher::hash(&final_key)
let module_prefix_hashed = Twox128::hash(Self::module_prefix());
let storage_prefix_hashed = Twox128::hash(Self::storage_prefix());
let key_hashed = key.borrow().using_encoded(Self::Hasher::hash);
let mut final_key = Vec::with_capacity(
module_prefix_hashed.len() + storage_prefix_hashed.len() + key_hashed.as_ref().len()
);
final_key.extend_from_slice(&module_prefix_hashed[..]);
final_key.extend_from_slice(&storage_prefix_hashed[..]);
final_key.extend_from_slice(key_hashed.as_ref());
final_key
}
}
@@ -62,7 +75,7 @@ impl<K: FullEncode, V: FullCodec, G: StorageMap<K, V>> storage::StorageMap<K, V>
type Query = G::Query;
fn hashed_key_for<KeyArg: EncodeLike<K>>(key: KeyArg) -> Vec<u8> {
Self::storage_map_final_key(key).as_ref().to_vec()
Self::storage_map_final_key(key)
}
fn swap<KeyArg1: EncodeLike<K>, KeyArg2: EncodeLike<K>>(key1: KeyArg1, key2: KeyArg2) {
@@ -94,22 +94,18 @@ mod tests {
let t = GenesisConfig::default().build_storage().unwrap();
TestExternalities::new(t).execute_with(|| {
let prefix = NumberMap::prefix();
// start with a map of u32 -> u32.
for i in 0u32..100u32 {
let final_key = <Format as KeyFormat>::storage_linked_map_final_key(
prefix, &i,
);
let final_key = <Format as KeyFormat>::storage_linked_map_final_key(&i);
let linkage = linked_map::new_head_linkage::<_, u32, u32, Format>(&i, prefix);
let linkage = linked_map::new_head_linkage::<_, u32, u32, Format>(&i);
unhashed::put(final_key.as_ref(), &(&i, linkage));
}
let head = linked_map::read_head::<u32, Format>().unwrap();
assert_eq!(
Enumerator::<u32, u32, Format>::from_head(head, prefix).collect::<Vec<_>>(),
Enumerator::<u32, u32, Format>::from_head(head).collect::<Vec<_>>(),
(0..100).rev().map(|x| (x, x)).collect::<Vec<_>>(),
);
@@ -21,16 +21,19 @@ use crate::{storage::{self, unhashed}, hash::{Twox128, StorageHasher}, traits::L
/// Generator for `StorageValue` used by `decl_storage`.
///
/// Value is stored at:
/// By default value is stored at:
/// ```nocompile
/// Twox128(unhashed_key)
/// Twox128(module_prefix) ++ Twox128(storage_prefix)
/// ```
pub trait StorageValue<T: FullCodec> {
/// The type that get/take returns.
type Query;
/// Unhashed key used in storage
fn unhashed_key() -> &'static [u8];
/// Module prefix. Used for generating final key.
fn module_prefix() -> &'static [u8];
/// Storage prefix. Used for generating final key.
fn storage_prefix() -> &'static [u8];
/// Convert an optional value retrieved from storage to the type queried.
fn from_optional_value_to_query(v: Option<T>) -> Self::Query;
@@ -39,15 +42,18 @@ pub trait StorageValue<T: FullCodec> {
fn from_query_to_optional_value(v: Self::Query) -> Option<T>;
/// Generate the full key used in top storage.
fn storage_value_final_key() -> [u8; 16] {
Twox128::hash(Self::unhashed_key())
fn storage_value_final_key() -> [u8; 32] {
let mut final_key = [0u8; 32];
final_key[0..16].copy_from_slice(&Twox128::hash(Self::module_prefix()));
final_key[16..32].copy_from_slice(&Twox128::hash(Self::storage_prefix()));
final_key
}
}
impl<T: FullCodec, G: StorageValue<T>> storage::StorageValue<T> for G {
type Query = G::Query;
fn hashed_key() -> [u8; 16] {
fn hashed_key() -> [u8; 32] {
Self::storage_value_final_key()
}
+1 -1
View File
@@ -34,7 +34,7 @@ pub trait StorageValue<T: FullCodec> {
type Query;
/// Get the storage key.
fn hashed_key() -> [u8; 16];
fn hashed_key() -> [u8; 32];
/// Does the value (explicitly) exist in storage?
fn exists() -> bool;