Use storage::append in the implementation of the storage types (#5889)

* Start improving `storage_append`

* Fix some stuff

* Fix compilation

* Update docs and add new test

* More tests

* Test kill + append
This commit is contained in:
Bastian Köcher
2020-05-05 15:09:07 +02:00
committed by GitHub
parent be0c7b9340
commit bc9707b4c5
19 changed files with 252 additions and 362 deletions
+43 -42
View File
@@ -126,15 +126,15 @@ dependencies = [
[[package]]
name = "arbitrary"
version = "0.4.2"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1148c9b25d393a07c4cc3ef5dd30f82a40a1c261018c4a670611ed8e76cad3ea"
checksum = "75153c95fdedd7db9732dfbfc3702324a1627eec91ba56e37cd0ac78314ab2ed"
[[package]]
name = "arc-swap"
version = "0.4.6"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b585a98a234c46fc563103e9278c9391fde1f4e6850334da895d27edb9580f62"
checksum = "d663a8e9a99154b5fb793032533f6328da35e23aac63d5c152279aa8ba356825"
[[package]]
name = "arrayref"
@@ -280,9 +280,9 @@ dependencies = [
[[package]]
name = "backtrace-sys"
version = "0.1.36"
version = "0.1.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78848718ee1255a2485d1309ad9cdecfc2e7d0362dd11c6829364c6b35ae1bc7"
checksum = "7de8aba10a69c8e8d7622c5710229485ec32e9d55fdad160ea559c086fdcd118"
dependencies = [
"cc",
"libc",
@@ -452,9 +452,9 @@ dependencies = [
[[package]]
name = "bs58"
version = "0.3.1"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "476e9cd489f9e121e02ffa6014a8ef220ecb15c05ed23fc34cca13925dc283fb"
checksum = "b170cd256a3f9fa6b9edae3e44a7dfdfc77e8124dbc3e2612d75f9c3e2396dae"
[[package]]
name = "bstr"
@@ -553,9 +553,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.0.51"
version = "1.0.50"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c9384ca4b90c0ea47e19a5c996d6643a3e73dedf9b89c65efb67587e34da1bb"
checksum = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd"
dependencies = [
"jobserver",
]
@@ -1147,18 +1147,18 @@ dependencies = [
[[package]]
name = "enumflags2"
version = "0.6.4"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83c8d82922337cd23a15f88b70d8e4ef5f11da38dd7cdb55e84dd5de99695da0"
checksum = "a80e524ebf194285b57e5e7944018721c7fffc673253f5183f7accd88a2a3b0c"
dependencies = [
"enumflags2_derive",
]
[[package]]
name = "enumflags2_derive"
version = "0.6.4"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "946ee94e3dbf58fdd324f9ce245c7b238d46a66f00e86a020b71996349e46cce"
checksum = "2ed9afacaea0301eefb738c9deea725e6d53938004597cdc518a8cf9a7aa2f03"
dependencies = [
"proc-macro2",
"quote 1.0.3",
@@ -2001,9 +2001,9 @@ dependencies = [
[[package]]
name = "hermit-abi"
version = "0.1.11"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a0d737e0f947a1864e93d33fdef4af8445a00d1ed8dc0c8ddb73139ea6abf15"
checksum = "725cf19794cf90aa94e65050cb4191ff5d8fa87a498383774c47b332e3af952e"
dependencies = [
"libc",
]
@@ -2156,9 +2156,9 @@ dependencies = [
[[package]]
name = "hyper"
version = "0.13.5"
version = "0.13.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96816e1d921eca64d208a85aab4f7798455a8e34229ee5a88c935bdee1b78b14"
checksum = "ed6081100e960d9d74734659ffc9cc91daf1c0fc7aceb8eaa94ee1a3f5046f2e"
dependencies = [
"bytes 0.5.4",
"futures-channel",
@@ -2187,7 +2187,7 @@ dependencies = [
"bytes 0.5.4",
"ct-logs",
"futures-util",
"hyper 0.13.5",
"hyper 0.13.4",
"log",
"rustls",
"rustls-native-certs",
@@ -3857,9 +3857,9 @@ dependencies = [
[[package]]
name = "num_cpus"
version = "1.13.0"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
checksum = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6"
dependencies = [
"hermit-abi",
"libc",
@@ -4880,7 +4880,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e"
dependencies = [
"lock_api",
"parking_lot_core 0.7.2",
"parking_lot_core 0.7.1",
]
[[package]]
@@ -4900,9 +4900,9 @@ dependencies = [
[[package]]
name = "parking_lot_core"
version = "0.7.2"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3"
checksum = "0e136c1904604defe99ce5fd71a28d473fa60a12255d511aa78a9ddf11237aeb"
dependencies = [
"cfg-if",
"cloudabi",
@@ -5604,9 +5604,9 @@ dependencies = [
[[package]]
name = "regex"
version = "1.3.7"
version = "1.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692"
checksum = "7f6946991529684867e47d86474e3a6d0c0ab9b82d5821e314b1ede31fa3a4b3"
dependencies = [
"aho-corasick",
"memchr",
@@ -6545,7 +6545,7 @@ dependencies = [
"fnv",
"futures 0.3.4",
"futures-timer 3.0.2",
"hyper 0.13.5",
"hyper 0.13.4",
"hyper-rustls",
"log",
"num_cpus",
@@ -7959,9 +7959,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
[[package]]
name = "structopt"
version = "0.3.14"
version = "0.3.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "863246aaf5ddd0d6928dfeb1a9ca65f505599e4e1b399935ef7e75107516b4ef"
checksum = "ff6da2e8d107dfd7b74df5ef4d205c6aebee0706c647f6bc6a2d5789905c00fb"
dependencies = [
"clap",
"lazy_static",
@@ -7970,9 +7970,9 @@ dependencies = [
[[package]]
name = "structopt-derive"
version = "0.4.7"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d239ca4b13aee7a2142e6795cbd69e457665ff8037aed33b3effdc430d2f927a"
checksum = "a489c87c08fbaf12e386665109dd13470dcc9c4583ea3e10dd2b4523e5ebd9ac"
dependencies = [
"heck",
"proc-macro-error",
@@ -8122,7 +8122,7 @@ dependencies = [
"async-std",
"derive_more",
"futures-util",
"hyper 0.13.5",
"hyper 0.13.4",
"log",
"prometheus",
"tokio 0.2.18",
@@ -8404,9 +8404,9 @@ dependencies = [
[[package]]
name = "sysinfo"
version = "0.13.4"
version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cac193374347e7c263c5f547524f36ff8ec6702d56c8799c8331d26dffe8c1e"
checksum = "5a0338198966bde7feb14b011a33d404a62a6e03b843352c71512a2a002634b7"
dependencies = [
"cfg-if",
"doc-comment",
@@ -8514,11 +8514,12 @@ dependencies = [
[[package]]
name = "time"
version = "0.1.43"
version = "0.1.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438"
checksum = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f"
dependencies = [
"libc",
"redox_syscall",
"winapi 0.3.8",
]
@@ -9379,18 +9380,18 @@ dependencies = [
[[package]]
name = "wast"
version = "14.0.0"
version = "13.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47b11c94c63d5365a76ea287f8e6e5b6050233fae4b2423aea2a1e126a385e17"
checksum = "5b20abd8b4a26f7e0d4dd5e357e90a3d555ec190e94472c9b2b27c5b9777f9ae"
dependencies = [
"leb128",
]
[[package]]
name = "wat"
version = "1.0.15"
version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03db18bc33cff3859c296efbefdcc00763a644539feeadca3415a1cee8a2835d"
checksum = "51a615830ee3e7200b505c441fec09aac2f114deae69df52f215cb828ba112c4"
dependencies = [
"wast",
]
@@ -9472,9 +9473,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.5"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
checksum = "fa515c5163a99cc82bab70fd3bfdd36d827be85de63737b40fcef2ce084a436e"
dependencies = [
"winapi 0.3.8",
]
+1 -1
View File
@@ -160,7 +160,7 @@ decl_module! {
#[weight = 0]
pub fn append_member_list(origin) {
let who = ensure_signed(origin)?;
MyMemberList::<T>::append(&[who])?;
MyMemberList::<T>::append(who);
}
/// Encode a vector of accounts to bytes.
+2 -3
View File
@@ -168,7 +168,7 @@ use sp_runtime::{
DispatchResult, DispatchError, RuntimeDebug,
traits::{Zero, Hash, Dispatchable, Saturating},
};
use codec::{Ref, Encode, Decode};
use codec::{Encode, Decode};
use frame_support::{
decl_module, decl_storage, decl_event, decl_error, ensure, Parameter,
weights::{Weight, DispatchClass},
@@ -553,8 +553,7 @@ decl_module! {
PublicPropCount::put(index + 1);
<DepositOf<T>>::insert(index, (value, &[&who][..]));
let new_prop = (index, proposal_hash, who);
<PublicProps<T>>::append_or_put(&[Ref::from(&new_prop)][..]);
<PublicProps<T>>::append((index, proposal_hash, who));
Self::deposit_event(RawEvent::Proposed(index, value));
}
+1 -1
View File
@@ -886,7 +886,7 @@ impl<T: Trait> Module<T> {
if set_len + 1 == VOTER_SET_SIZE {
NextVoterSet::put(next + 1);
}
<Voters<T>>::append_or_insert(next, &[Some(who.clone())][..])
<Voters<T>>::append(next, Some(who.clone()));
}
}
+12 -12
View File
@@ -52,7 +52,7 @@ use frame_support::{
traits::{Get, schedule},
weights::{GetDispatchInfo, Weight},
};
use frame_system::{self as system};
use frame_system as system;
/// Our pallet's configuration trait. All our types and constants go in here. If the
/// pallet is dependent on specific other pallets, then their configuration traits
@@ -123,7 +123,7 @@ decl_module! {
.collect::<Vec<_>>();
queued.sort_by_key(|(_, s)| s.priority);
let mut result = 0;
let unused_items = queued.into_iter()
queued.into_iter()
.enumerate()
.scan(0, |cumulative_weight, (order, (index, s))| {
*cumulative_weight += s.call.get_dispatch_info().weight;
@@ -141,10 +141,10 @@ decl_module! {
}
let next = now + period;
if let Some(ref id) = s.maybe_id {
let next_index = Agenda::<T>::decode_len(now + period).unwrap_or(0) as u32;
Lookup::<T>::insert(id, (next, next_index));
let next_index = Agenda::<T>::decode_len(now + period).unwrap_or(0);
Lookup::<T>::insert(id, (next, next_index as u32));
}
Agenda::<T>::append_or_insert(next, &[Some(s)][..]);
Agenda::<T>::append(next, Some(s));
} else {
if let Some(ref id) = s.maybe_id {
Lookup::<T>::remove(id);
@@ -161,11 +161,11 @@ decl_module! {
Some(Some(s))
}
})
.collect::<Vec<_>>();
if !unused_items.is_empty() {
let next = now + One::one();
Agenda::<T>::append_or_insert(next, &unused_items[..]);
}
.for_each(|unused| {
let next = now + One::one();
Agenda::<T>::append(next, unused);
});
result
}
}
@@ -186,7 +186,7 @@ impl<T: Trait> schedule::Anon<T::BlockNumber, <T as Trait>::Call> for Module<T>
// Remove one from the number of repetitions since we will schedule one now.
.map(|(p, c)| (p, c - 1));
let s = Some(Scheduled { maybe_id: None, priority, call, maybe_periodic });
Agenda::<T>::append_or_insert(when, &[s][..]);
Agenda::<T>::append(when, s);
(when, Agenda::<T>::decode_len(when).unwrap_or(1) as u32 - 1)
}
@@ -225,7 +225,7 @@ impl<T: Trait> schedule::Named<T::BlockNumber, <T as Trait>::Call> for Module<T>
.map(|(p, c)| (p, c - 1));
let s = Scheduled { maybe_id: Some(id.clone()), priority, call, maybe_periodic };
Agenda::<T>::append_or_insert(when, &[Some(s)][..]);
Agenda::<T>::append(when, Some(s));
let index = Agenda::<T>::decode_len(when).unwrap_or(1) as u32 - 1;
let address = (when, index);
Lookup::<T>::insert(&id, &address);
+2 -7
View File
@@ -100,9 +100,7 @@ use frame_support::{
weights::Weight,
};
use frame_system::{self as system, ensure_root, ensure_signed};
use sp_runtime::{
traits::{AtLeast32Bit, MaybeSerializeDeserialize, Zero, StaticLookup},
};
use sp_runtime::traits::{AtLeast32Bit, MaybeSerializeDeserialize, Zero, StaticLookup};
type BalanceOf<T, I> = <<T as Trait<I>>::Currency as Currency<<T as frame_system::Trait>::AccountId>>::Balance;
type PoolT<T, I> = Vec<(<T as frame_system::Trait>::AccountId, Option<<T as Trait<I>>::Score>)>;
@@ -276,10 +274,7 @@ decl_module! {
// can be inserted as last element in pool, since entities with
// `None` are always sorted to the end.
if let Err(e) = <Pool<T, I>>::append(&[(who.clone(), None)]) {
T::Currency::unreserve(&who, deposit);
Err(e)?
}
<Pool<T, I>>::append((who.clone(), Option::<<T as Trait<I>>::Score>::None));
<CandidateExists<T, I>>::insert(&who, true);
+2 -2
View File
@@ -499,8 +499,8 @@ mod tests {
let key2 = 18u32;
DoubleMap::insert(&key1, &key2, &vec![1]);
DoubleMap::append(&key1, &key2, &[2, 3]).unwrap();
assert_eq!(DoubleMap::get(&key1, &key2), &[1, 2, 3]);
DoubleMap::append(&key1, &key2, 2);
assert_eq!(DoubleMap::get(&key1, &key2), &[1, 2]);
});
}
@@ -16,8 +16,8 @@
use sp_std::prelude::*;
use sp_std::borrow::Borrow;
use codec::{Ref, FullCodec, FullEncode, Decode, Encode, EncodeLike, EncodeAppend};
use crate::{storage::{self, unhashed}, traits::Len, Never};
use codec::{FullCodec, FullEncode, Decode, Encode, EncodeLike};
use crate::{storage::{self, unhashed, StorageAppend}, traits::Len, Never};
use crate::hash::{StorageHasher, Twox128, ReversibleStorageHasher};
/// Generator for `StorageDoubleMap` used by `decl_storage`.
@@ -245,53 +245,19 @@ impl<K1, K2, V, G> storage::StorageDoubleMap<K1, K2, V> for G where
ret
}
fn append<Items, Item, EncodeLikeItem, KArg1, KArg2>(
fn append<Item, EncodeLikeItem, KArg1, KArg2>(
k1: KArg1,
k2: KArg2,
items: Items,
) -> Result<(), &'static str> where
KArg1: EncodeLike<K1>,
KArg2: EncodeLike<K2>,
Item: Encode,
EncodeLikeItem: EncodeLike<Item>,
V: EncodeAppend<Item=Item>,
Items: IntoIterator<Item=EncodeLikeItem>,
Items::IntoIter: ExactSizeIterator
{
let final_key = Self::storage_double_map_final_key(k1, k2);
let encoded_value = unhashed::get_raw(&final_key)
.unwrap_or_else(|| {
match G::from_query_to_optional_value(G::from_optional_value_to_query(None)) {
Some(value) => value.encode(),
None => Vec::new(),
}
});
let new_val = V::append_or_new(
encoded_value,
items,
).map_err(|_| "Could not append given item")?;
unhashed::put_raw(&final_key, &new_val);
Ok(())
}
fn append_or_insert<Items, Item, EncodeLikeItem, KArg1, KArg2>(
k1: KArg1,
k2: KArg2,
items: Items,
item: EncodeLikeItem,
) where
KArg1: EncodeLike<K1>,
KArg2: EncodeLike<K2>,
Item: Encode,
EncodeLikeItem: EncodeLike<Item>,
V: EncodeAppend<Item=Item>,
Items: IntoIterator<Item=EncodeLikeItem> + Clone + EncodeLike<V>,
Items::IntoIter: ExactSizeIterator
V: StorageAppend<Item>,
{
Self::append(Ref::from(&k1), Ref::from(&k2), items.clone())
.unwrap_or_else(|_| Self::insert(k1, k2, items));
let final_key = Self::storage_double_map_final_key(k1, k2);
sp_io::storage::append(&final_key, item.encode());
}
fn decode_len<KArg1, KArg2>(key1: KArg1, key2: KArg2) -> Result<usize, &'static str> where
@@ -17,8 +17,8 @@
#[cfg(not(feature = "std"))]
use sp_std::prelude::*;
use sp_std::borrow::Borrow;
use codec::{FullCodec, FullEncode, Decode, Encode, EncodeLike, Ref, EncodeAppend};
use crate::{storage::{self, unhashed}, traits::Len, Never};
use codec::{FullCodec, FullEncode, Decode, Encode, EncodeLike};
use crate::{storage::{self, unhashed, StorageAppend}, traits::Len, Never};
use crate::hash::{StorageHasher, Twox128, ReversibleStorageHasher};
/// Generator for `StorageMap` used by `decl_storage`.
@@ -276,43 +276,15 @@ impl<K: FullEncode, V: FullCodec, G: StorageMap<K, V>> storage::StorageMap<K, V>
G::from_optional_value_to_query(value)
}
fn append<Items, Item, EncodeLikeItem, KeyArg>(key: KeyArg, items: Items) -> Result<(), &'static str>
fn append<Item, EncodeLikeItem, EncodeLikeKey>(key: EncodeLikeKey, item: EncodeLikeItem)
where
KeyArg: EncodeLike<K>,
EncodeLikeKey: EncodeLike<K>,
Item: Encode,
EncodeLikeItem: EncodeLike<Item>,
V: EncodeAppend<Item=Item>,
Items: IntoIterator<Item=EncodeLikeItem>,
Items::IntoIter: ExactSizeIterator,
V: StorageAppend<Item>,
{
let key = Self::storage_map_final_key(key);
let encoded_value = unhashed::get_raw(key.as_ref())
.unwrap_or_else(|| {
match G::from_query_to_optional_value(G::from_optional_value_to_query(None)) {
Some(value) => value.encode(),
None => Vec::new(),
}
});
let new_val = V::append_or_new(
encoded_value,
items,
).map_err(|_| "Could not append given item")?;
unhashed::put_raw(key.as_ref(), &new_val);
Ok(())
}
fn append_or_insert<Items, Item, EncodeLikeItem, KeyArg>(key: KeyArg, items: Items)
where
KeyArg: EncodeLike<K>,
Item: Encode,
EncodeLikeItem: EncodeLike<Item>,
V: EncodeAppend<Item=Item>,
Items: IntoIterator<Item=EncodeLikeItem> + Clone + EncodeLike<V>,
Items::IntoIter: ExactSizeIterator,
{
Self::append(Ref::from(&key), items.clone())
.unwrap_or_else(|_| Self::insert(key, items));
sp_io::storage::append(&key, item.encode());
}
fn decode_len<KeyArg: EncodeLike<K>>(key: KeyArg) -> Result<usize, &'static str>
@@ -14,12 +14,10 @@
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
#[cfg(not(feature = "std"))]
use sp_std::prelude::*;
use codec::{FullCodec, Encode, EncodeAppend, EncodeLike, Decode};
use codec::{FullCodec, Encode, EncodeLike, Decode};
use crate::{
Never,
storage::{self, unhashed},
storage::{self, unhashed, StorageAppend},
hash::{Twox128, StorageHasher},
traits::Len
};
@@ -134,55 +132,16 @@ impl<T: FullCodec, G: StorageValue<T>> storage::StorageValue<T> for G {
G::from_optional_value_to_query(value)
}
/// Append the given items to the value in the storage.
///
/// `T` is required to implement `codec::EncodeAppend`.
fn append<Items, Item, EncodeLikeItem>(items: Items) -> Result<(), &'static str>
fn append<Item, EncodeLikeItem>(item: EncodeLikeItem)
where
Item: Encode,
EncodeLikeItem: EncodeLike<Item>,
T: EncodeAppend<Item=Item>,
Items: IntoIterator<Item=EncodeLikeItem>,
Items::IntoIter: ExactSizeIterator,
T: StorageAppend<Item>,
{
let key = Self::storage_value_final_key();
let encoded_value = unhashed::get_raw(&key)
.unwrap_or_else(|| {
match G::from_query_to_optional_value(G::from_optional_value_to_query(None)) {
Some(value) => value.encode(),
None => Vec::new(),
}
});
let new_val = T::append_or_new(
encoded_value,
items,
).map_err(|_| "Could not append given item")?;
unhashed::put_raw(&key, &new_val);
Ok(())
sp_io::storage::append(&key, item.encode());
}
/// Safely append the given items to the value in the storage. If a codec error occurs, then the
/// old (presumably corrupt) value is replaced with the given `items`.
///
/// `T` is required to implement `codec::EncodeAppend`.
fn append_or_put<Items, Item, EncodeLikeItem>(items: Items) where
Item: Encode,
EncodeLikeItem: EncodeLike<Item>,
T: EncodeAppend<Item=Item>,
Items: IntoIterator<Item=EncodeLikeItem> + Clone + EncodeLike<T>,
Items::IntoIter: ExactSizeIterator
{
Self::append(items.clone()).unwrap_or_else(|_| Self::put(items));
}
/// Read the length of the value in a fast way, without decoding the entire value.
///
/// `T` is required to implement `Codec::DecodeLength`.
///
/// Note that `0` is returned as the default value if no encoded value exists at the given key.
/// Therefore, this function cannot be used as a sign of _existence_. use the `::exists()`
/// function for this purpose.
fn decode_len() -> Result<usize, &'static str> where T: codec::DecodeLength, T: Len {
let key = Self::storage_value_final_key();
+45 -67
View File
@@ -17,7 +17,7 @@
//! Stuff to do with the runtime's storage.
use sp_std::{prelude::*, marker::PhantomData};
use codec::{FullCodec, FullEncode, Encode, EncodeAppend, EncodeLike, Decode};
use codec::{FullCodec, FullEncode, Encode, EncodeLike, Decode};
use crate::{traits::Len, hash::{Twox128, StorageHasher}};
pub mod unhashed;
@@ -91,33 +91,18 @@ pub trait StorageValue<T: FullCodec> {
/// Append the given item to the value in the storage.
///
/// `T` is required to implement `codec::EncodeAppend`.
fn append<Items, Item, EncodeLikeItem>(items: Items) -> Result<(), &'static str>
/// `T` is required to implement [`StorageAppend`].
///
/// # Warning
///
/// If the storage item is not encoded properly, the storage item will be overwritten
/// and set to `[item]`. Any default value set for the storage item will be ignored
/// on overwrite.
fn append<Item, EncodeLikeItem>(item: EncodeLikeItem)
where
Item: Encode,
EncodeLikeItem: EncodeLike<Item>,
T: EncodeAppend<Item=Item>,
Items: IntoIterator<Item=EncodeLikeItem>,
Items::IntoIter: ExactSizeIterator;
/// Append the given items to the value in the storage.
///
/// `T` is required to implement `Codec::EncodeAppend`.
///
/// Upon any failure, it replaces `items` as the new value (assuming that the previous stored
/// data is simply corrupt and no longer usable).
///
/// ### WARNING
///
/// use with care; if your use-case is not _exactly_ as what this function is doing,
/// you should use append and sensibly handle failure within the runtime code if it happens.
fn append_or_put<Items, Item, EncodeLikeItem>(items: Items) where
Item: Encode,
EncodeLikeItem: EncodeLike<Item>,
T: EncodeAppend<Item=Item>,
Items: IntoIterator<Item=EncodeLikeItem> + Clone + EncodeLike<T>,
Items::IntoIter: ExactSizeIterator;
T: StorageAppend<Item>;
/// Read the length of the value in a fast way, without decoding the entire value.
///
@@ -176,25 +161,18 @@ pub trait StorageMap<K: FullEncode, V: FullCodec> {
/// Append the given items to the value in the storage.
///
/// `V` is required to implement `codec::EncodeAppend`.
fn append<Items, Item, EncodeLikeItem, KeyArg>(key: KeyArg, items: Items) -> Result<(), &'static str> where
KeyArg: EncodeLike<K>,
Item: Encode,
EncodeLikeItem: EncodeLike<Item>,
V: EncodeAppend<Item=Item>,
Items: IntoIterator<Item=EncodeLikeItem>,
Items::IntoIter: ExactSizeIterator;
/// Safely append the given items to the value in the storage. If a codec error occurs, then the
/// old (presumably corrupt) value is replaced with the given `items`.
///
/// `V` is required to implement `codec::EncodeAppend`.
fn append_or_insert<Items, Item, EncodeLikeItem, KeyArg>(key: KeyArg, items: Items) where
KeyArg: EncodeLike<K>,
/// # Warning
///
/// If the storage item is not encoded properly, the storage will be overwritten
/// and set to `[item]`. Any default value set for the storage item will be ignored
/// on overwrite.
fn append<Item, EncodeLikeItem, EncodeLikeKey>(key: EncodeLikeKey, item: EncodeLikeItem)
where
EncodeLikeKey: EncodeLike<K>,
Item: Encode,
EncodeLikeItem: EncodeLike<Item>,
V: EncodeAppend<Item=Item>,
Items: IntoIterator<Item=EncodeLikeItem> + Clone + EncodeLike<V>,
Items::IntoIter: ExactSizeIterator;
V: StorageAppend<Item>;
/// Read the length of the value in a fast way, without decoding the entire value.
///
@@ -351,38 +329,23 @@ pub trait StorageDoubleMap<K1: FullEncode, K2: FullEncode, V: FullCodec> {
/// Append the given item to the value in the storage.
///
/// `V` is required to implement `codec::EncodeAppend`.
fn append<Items, Item, EncodeLikeItem, KArg1, KArg2>(
k1: KArg1,
k2: KArg2,
items: Items,
) -> Result<(), &'static str>
where
KArg1: EncodeLike<K1>,
KArg2: EncodeLike<K2>,
Item: Encode,
EncodeLikeItem: EncodeLike<Item>,
V: EncodeAppend<Item=Item>,
Items: IntoIterator<Item=EncodeLikeItem>,
Items::IntoIter: ExactSizeIterator;
/// Safely append the given items to the value in the storage. If a codec error occurs, then the
/// old (presumably corrupt) value is replaced with the given `items`.
/// `V` is required to implement [`StorageAppend`].
///
/// `V` is required to implement `codec::EncodeAppend`.
fn append_or_insert<Items, Item, EncodeLikeItem, KArg1, KArg2>(
/// # Warning
///
/// If the storage item is not encoded properly, the storage will be overwritten
/// and set to `[item]`. Any default value set for the storage item will be ignored
/// on overwrite.
fn append<Item, EncodeLikeItem, KArg1, KArg2>(
k1: KArg1,
k2: KArg2,
items: Items,
)
where
item: EncodeLikeItem,
) where
KArg1: EncodeLike<K1>,
KArg2: EncodeLike<K2>,
Item: Encode,
EncodeLikeItem: EncodeLike<Item>,
V: EncodeAppend<Item=Item>,
Items: IntoIterator<Item=EncodeLikeItem> + Clone + EncodeLike<V>,
Items::IntoIter: ExactSizeIterator;
V: StorageAppend<Item>;
/// Read the length of the value in a fast way, without decoding the entire value.
///
@@ -449,7 +412,6 @@ impl<Value: Decode> Iterator for PrefixIterator<Value> {
/// Twox128(module_prefix) ++ Twox128(storage_prefix)
/// ```
pub trait StoragePrefixedMap<Value: FullCodec> {
/// Module prefix. Used for generating final key.
fn module_prefix() -> &'static [u8];
@@ -525,6 +487,22 @@ pub trait StoragePrefixedMap<Value: FullCodec> {
}
}
/// Marker trait that will be implemented for types that support the `storage::append` api.
///
/// This trait is sealed.
pub trait StorageAppend<Item: Encode>: private::Sealed {}
/// Provides `Sealed` trait to prevent implementing trait `StorageAppend` outside of this crate.
mod private {
use super::*;
pub trait Sealed {}
impl<T: Encode> Sealed for Vec<T> {}
}
impl<T: Encode> StorageAppend<T> for Vec<T> {}
#[cfg(test)]
mod test {
use sp_core::hashing::twox_128;
@@ -526,50 +526,58 @@ mod test_append_and_len {
#[test]
fn append_works() {
TestExternalities::default().execute_with(|| {
let _ = MapVec::append(1, [1, 2, 3].iter());
let _ = MapVec::append(1, [4, 5].iter());
for val in &[1, 2, 3, 4, 5] {
MapVec::append(1, val);
}
assert_eq!(MapVec::get(1), vec![1, 2, 3, 4, 5]);
let _ = JustVec::append([1, 2, 3].iter());
let _ = JustVec::append([4, 5].iter());
MapVec::remove(1);
MapVec::append(1, 1);
assert_eq!(MapVec::get(1), vec![1]);
for val in &[1, 2, 3, 4, 5] {
JustVec::append(val);
}
assert_eq!(JustVec::get(), vec![1, 2, 3, 4, 5]);
JustVec::kill();
JustVec::append(1);
assert_eq!(JustVec::get(), vec![1]);
});
}
#[test]
fn append_works_for_default() {
fn append_overwrites_invalid_data() {
TestExternalities::default().execute_with(|| {
let key = JustVec::hashed_key();
// Set it to some invalid value.
frame_support::storage::unhashed::put_raw(&key, &*b"1");
assert_eq!(JustVec::get(), Vec::new());
assert_eq!(frame_support::storage::unhashed::get_raw(&key), Some(b"1".to_vec()));
JustVec::append(1);
JustVec::append(2);
assert_eq!(JustVec::get(), vec![1, 2]);
});
}
#[test]
fn append_overwrites_default() {
TestExternalities::default().execute_with(|| {
assert_eq!(JustVecWithDefault::get(), vec![6, 9]);
let _ = JustVecWithDefault::append([1].iter());
assert_eq!(JustVecWithDefault::get(), vec![6, 9, 1]);
JustVecWithDefault::append(1);
assert_eq!(JustVecWithDefault::get(), vec![1]);
assert_eq!(MapVecWithDefault::get(0), vec![6, 9]);
let _ = MapVecWithDefault::append(0, [1].iter());
assert_eq!(MapVecWithDefault::get(0), vec![6, 9, 1]);
MapVecWithDefault::append(0, 1);
assert_eq!(MapVecWithDefault::get(0), vec![1]);
assert_eq!(OptionVec::get(), None);
let _ = OptionVec::append([1].iter());
OptionVec::append(1);
assert_eq!(OptionVec::get(), Some(vec![1]));
});
}
#[test]
fn append_or_put_works() {
TestExternalities::default().execute_with(|| {
let _ = MapVec::append_or_insert(1, &[1, 2, 3][..]);
let _ = MapVec::append_or_insert(1, &[4, 5][..]);
assert_eq!(MapVec::get(1), vec![1, 2, 3, 4, 5]);
let _ = JustVec::append_or_put(&[1, 2, 3][..]);
let _ = JustVec::append_or_put(&[4, 5][..]);
assert_eq!(JustVec::get(), vec![1, 2, 3, 4, 5]);
let _ = OptionVec::append_or_put(&[1, 2, 3][..]);
let _ = OptionVec::append_or_put(&[4, 5][..]);
assert_eq!(OptionVec::get(), Some(vec![1, 2, 3, 4, 5]));
});
}
#[test]
fn len_works() {
TestExternalities::default().execute_with(|| {
+3 -9
View File
@@ -116,7 +116,7 @@ use sp_runtime::{
use sp_core::{ChangesTrieConfiguration, storage::well_known_keys};
use frame_support::{
decl_module, decl_event, decl_storage, decl_error, Parameter, ensure, debug,
storage::{self, generator::StorageValue},
storage,
traits::{
Contains, Get, ModuleToIndex, OnNewAccount, OnKilledAccount, IsDeadAccount, Happened,
StoredMap, EnsureOrigin,
@@ -852,16 +852,10 @@ impl<T: Trait> Module<T> {
old_event_count
};
// We use append api here to avoid bringing all events in the runtime when we push a
// new one in the list.
let encoded_event = event.encode();
sp_io::storage::append(&Events::<T>::storage_value_final_key()[..], encoded_event);
Events::<T>::append(&event);
for topic in topics {
// The same applies here.
if <EventTopics<T>>::append(topic, &[(block_number, event_idx)]).is_err() {
return;
}
<EventTopics<T>>::append(topic, &(block_number, event_idx));
}
}
+2 -1
View File
@@ -25,7 +25,6 @@ use crate::{
};
#[cfg(feature = "std")]
use std::collections::HashSet;
use codec::Encode;
/// Key type for generic Ed25519 key.
pub const ED25519: KeyTypeId = KeyTypeId(*b"ed25");
/// Key type for generic Sr 25519 key.
@@ -172,6 +171,8 @@ impl crate::traits::BareCryptoStore for KeyStore {
key: &CryptoTypePublicPair,
msg: &[u8],
) -> Result<Vec<u8>, BareCryptoStoreError> {
use codec::Encode;
match key.0 {
ed25519::CRYPTO_ID => {
let key_pair: ed25519::Pair = self
+8 -1
View File
@@ -117,7 +117,14 @@ pub trait Storage {
Externalities::clear_prefix(*self, prefix)
}
/// Append to storage item (assumes it is in "Vec" format).
/// Append the encoded `value` to the storage item at `key`.
///
/// The storage item needs to implement [`EncodeAppend`](codec::EncodeAppend).
///
/// # Warning
///
/// If the storage item does not support [`EncodeAppend`](codec::EncodeAppend) or
/// something else fails at appending, the storage item will be set to `[value]`.
fn append(&mut self, key: &[u8], value: Vec<u8>) {
self.storage_append(key.to_vec(), value);
}
@@ -262,8 +262,8 @@ impl Externalities for BasicExternalities {
key: Vec<u8>,
value: Vec<u8>,
) {
let previous = self.inner.top.entry(key).or_default();
crate::ext::append_to_storage(previous, value).expect("Failed to append to storage");
let current = self.inner.top.entry(key).or_default();
crate::ext::StorageAppend::new(current).append(value);
}
fn chain_id(&self) -> u64 { 42 }
+63 -57
View File
@@ -30,7 +30,7 @@ use sp_core::{
};
use sp_trie::{trie_types::Layout, empty_child_trie_root};
use sp_externalities::{Extensions, Extension};
use codec::{Compact, Decode, Encode};
use codec::{Decode, Encode, EncodeAppend};
use std::{error, fmt, any::{Any, TypeId}};
use log::{warn, trace};
@@ -428,7 +428,7 @@ where
&key,
|| backend.storage(&key).expect(EXT_NOT_ALLOWED_TO_FAIL).unwrap_or_default()
);
append_to_storage(current_value, value).expect(EXT_NOT_ALLOWED_TO_FAIL);
StorageAppend::new(current_value).append(value);
}
fn chain_id(&self) -> u64 {
@@ -549,16 +549,24 @@ where
fn wipe(&mut self) {
self.overlay.discard_prospective();
self.overlay.drain_storage_changes(&self.backend, None, Default::default(), self.storage_transaction_cache)
.expect(EXT_NOT_ALLOWED_TO_FAIL);
self.overlay.drain_storage_changes(
&self.backend,
None,
Default::default(),
self.storage_transaction_cache,
).expect(EXT_NOT_ALLOWED_TO_FAIL);
self.storage_transaction_cache.reset();
self.backend.wipe().expect(EXT_NOT_ALLOWED_TO_FAIL)
}
fn commit(&mut self) {
self.overlay.commit_prospective();
let changes = self.overlay.drain_storage_changes(&self.backend, None, Default::default(), self.storage_transaction_cache)
.expect(EXT_NOT_ALLOWED_TO_FAIL);
let changes = self.overlay.drain_storage_changes(
&self.backend,
None,
Default::default(),
self.storage_transaction_cache,
).expect(EXT_NOT_ALLOWED_TO_FAIL);
self.backend.commit(
changes.transaction_storage_root,
changes.transaction,
@@ -567,65 +575,43 @@ where
}
}
fn extract_length_data(data: &[u8]) -> Result<(u32, usize, usize), &'static str> {
use codec::CompactLen;
let len = u32::from(
Compact::<u32>::decode(&mut &data[..])
.map_err(|_| "Incorrect updated item encoding")?
);
let new_len = len
.checked_add(1)
.ok_or_else(|| "New vec length greater than `u32::max_value()`.")?;
/// Implement `Encode` by forwarding the stored raw vec.
struct EncodeOpaqueValue(Vec<u8>);
let encoded_len = Compact::<u32>::compact_len(&len);
let encoded_new_len = Compact::<u32>::compact_len(&new_len);
Ok((new_len, encoded_len, encoded_new_len))
impl Encode for EncodeOpaqueValue {
fn using_encoded<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
f(&self.0)
}
}
pub fn append_to_storage(
self_encoded: &mut Vec<u8>,
value: Vec<u8>,
) -> Result<(), &'static str> {
// No data present, just encode the given input data.
if self_encoded.is_empty() {
Compact::from(1u32).encode_to(self_encoded);
self_encoded.extend(value);
return Ok(());
/// Auxialiary structure for appending a value to a storage item.
pub(crate) struct StorageAppend<'a>(&'a mut Vec<u8>);
impl<'a> StorageAppend<'a> {
/// Create a new instance using the given `storage` reference.
pub fn new(storage: &'a mut Vec<u8>) -> Self {
Self(storage)
}
let (new_len, encoded_len, encoded_new_len) = extract_length_data(&self_encoded)?;
/// Append the given `value` to the storage item.
///
/// If appending fails, `[value]` is stored in the storage item.
pub fn append(&mut self, value: Vec<u8>) {
let value = vec![EncodeOpaqueValue(value)];
let replace_len = |dest: &mut Vec<u8>| {
Compact(new_len).using_encoded(|e| {
dest[..encoded_new_len].copy_from_slice(e);
})
};
let item = std::mem::take(self.0);
let append_new_elems = |dest: &mut Vec<u8>| dest.extend(&value[..]);
// If old and new encoded len is equal, we don't need to copy the
// already encoded data.
if encoded_len == encoded_new_len {
replace_len(self_encoded);
append_new_elems(self_encoded);
Ok(())
} else {
let size = encoded_new_len + self_encoded.len() - encoded_len;
let mut res = Vec::with_capacity(size + value.len());
unsafe { res.set_len(size); }
// Insert the new encoded len, copy the already encoded data and
// add the new element.
replace_len(&mut res);
res[encoded_new_len..size].copy_from_slice(&self_encoded[encoded_len..]);
append_new_elems(&mut res);
*self_encoded = res;
Ok(())
*self.0 = match Vec::<EncodeOpaqueValue>::append_or_new(item, &value) {
Ok(item) => item,
Err(_) => {
log::error!(
target: "runtime",
"Failed to append value, resetting storage item to `[value]`.",
);
value.encode()
}
};
}
}
@@ -902,4 +888,24 @@ mod tests {
Some(Blake2Hasher::hash(&[31]).as_ref().to_vec()),
);
}
#[test]
fn storage_append_works() {
let mut data = Vec::new();
let mut append = StorageAppend::new(&mut data);
append.append(1u32.encode());
append.append(2u32.encode());
drop(append);
assert_eq!(Vec::<u32>::decode(&mut &data[..]).unwrap(), vec![1, 2]);
// Initialize with some invalid data
let mut data = vec![1];
let mut append = StorageAppend::new(&mut data);
append.append(1u32.encode());
append.append(2u32.encode());
drop(append);
assert_eq!(Vec::<u32>::decode(&mut &data[..]).unwrap(), vec![1, 2]);
}
}
+12 -8
View File
@@ -22,8 +22,7 @@
use serde::{Serialize, Deserialize};
use sp_debug_derive::RuntimeDebug;
use sp_std::vec::Vec;
use sp_std::ops::{Deref, DerefMut};
use sp_std::{vec::Vec, ops::{Deref, DerefMut}};
use ref_cast::RefCast;
/// Storage key.
@@ -334,10 +333,15 @@ impl ChildTrieParentKeyId {
}
}
#[test]
fn test_prefix_default_child_info() {
let child_info = ChildInfo::new_default(b"any key");
let prefix = child_info.child_type().parent_prefix();
assert!(prefix.starts_with(well_known_keys::CHILD_STORAGE_KEY_PREFIX));
assert!(prefix.starts_with(well_known_keys::DEFAULT_CHILD_STORAGE_KEY_PREFIX));
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_prefix_default_child_info() {
let child_info = ChildInfo::new_default(b"any key");
let prefix = child_info.child_type().parent_prefix();
assert!(prefix.starts_with(well_known_keys::CHILD_STORAGE_KEY_PREFIX));
assert!(prefix.starts_with(well_known_keys::DEFAULT_CHILD_STORAGE_KEY_PREFIX));
}
}
@@ -119,7 +119,7 @@ mod private {
/// Something that can be wrapped in a wasm `Pointer`.
///
/// This trait is sealed.
pub trait PointerType: Sized {
pub trait PointerType: Sized + private::Sealed {
/// The size of the type in wasm.
const SIZE: u32 = mem::size_of::<Self>() as u32;
}