diff --git a/substrate/Cargo.lock b/substrate/Cargo.lock index 64e126f739..3fce87f7aa 100644 --- a/substrate/Cargo.lock +++ b/substrate/Cargo.lock @@ -126,15 +126,15 @@ dependencies = [ [[package]] name = "arbitrary" -version = "0.4.2" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1148c9b25d393a07c4cc3ef5dd30f82a40a1c261018c4a670611ed8e76cad3ea" +checksum = "75153c95fdedd7db9732dfbfc3702324a1627eec91ba56e37cd0ac78314ab2ed" [[package]] name = "arc-swap" -version = "0.4.6" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b585a98a234c46fc563103e9278c9391fde1f4e6850334da895d27edb9580f62" +checksum = "d663a8e9a99154b5fb793032533f6328da35e23aac63d5c152279aa8ba356825" [[package]] name = "arrayref" @@ -280,9 +280,9 @@ dependencies = [ [[package]] name = "backtrace-sys" -version = "0.1.36" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78848718ee1255a2485d1309ad9cdecfc2e7d0362dd11c6829364c6b35ae1bc7" +checksum = "7de8aba10a69c8e8d7622c5710229485ec32e9d55fdad160ea559c086fdcd118" dependencies = [ "cc", "libc", @@ -452,9 +452,9 @@ dependencies = [ [[package]] name = "bs58" -version = "0.3.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "476e9cd489f9e121e02ffa6014a8ef220ecb15c05ed23fc34cca13925dc283fb" +checksum = "b170cd256a3f9fa6b9edae3e44a7dfdfc77e8124dbc3e2612d75f9c3e2396dae" [[package]] name = "bstr" @@ -553,9 +553,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.51" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9384ca4b90c0ea47e19a5c996d6643a3e73dedf9b89c65efb67587e34da1bb" +checksum = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" dependencies = [ "jobserver", ] @@ -1147,18 +1147,18 @@ dependencies = [ [[package]] name = "enumflags2" -version = "0.6.4" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83c8d82922337cd23a15f88b70d8e4ef5f11da38dd7cdb55e84dd5de99695da0" +checksum = "a80e524ebf194285b57e5e7944018721c7fffc673253f5183f7accd88a2a3b0c" dependencies = [ "enumflags2_derive", ] [[package]] name = "enumflags2_derive" -version = "0.6.4" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "946ee94e3dbf58fdd324f9ce245c7b238d46a66f00e86a020b71996349e46cce" +checksum = "2ed9afacaea0301eefb738c9deea725e6d53938004597cdc518a8cf9a7aa2f03" dependencies = [ "proc-macro2", "quote 1.0.3", @@ -2001,9 +2001,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.11" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d737e0f947a1864e93d33fdef4af8445a00d1ed8dc0c8ddb73139ea6abf15" +checksum = "725cf19794cf90aa94e65050cb4191ff5d8fa87a498383774c47b332e3af952e" dependencies = [ "libc", ] @@ -2156,9 +2156,9 @@ dependencies = [ [[package]] name = "hyper" -version = "0.13.5" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96816e1d921eca64d208a85aab4f7798455a8e34229ee5a88c935bdee1b78b14" +checksum = "ed6081100e960d9d74734659ffc9cc91daf1c0fc7aceb8eaa94ee1a3f5046f2e" dependencies = [ "bytes 0.5.4", "futures-channel", @@ -2187,7 +2187,7 @@ dependencies = [ "bytes 0.5.4", "ct-logs", "futures-util", - "hyper 0.13.5", + "hyper 0.13.4", "log", "rustls", "rustls-native-certs", @@ -3857,9 +3857,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.13.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +checksum = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6" dependencies = [ "hermit-abi", "libc", @@ -4880,7 +4880,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" dependencies = [ "lock_api", - "parking_lot_core 0.7.2", + "parking_lot_core 0.7.1", ] [[package]] @@ -4900,9 +4900,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.7.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" +checksum = "0e136c1904604defe99ce5fd71a28d473fa60a12255d511aa78a9ddf11237aeb" dependencies = [ "cfg-if", "cloudabi", @@ -5604,9 +5604,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.3.7" +version = "1.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692" +checksum = "7f6946991529684867e47d86474e3a6d0c0ab9b82d5821e314b1ede31fa3a4b3" dependencies = [ "aho-corasick", "memchr", @@ -6545,7 +6545,7 @@ dependencies = [ "fnv", "futures 0.3.4", "futures-timer 3.0.2", - "hyper 0.13.5", + "hyper 0.13.4", "hyper-rustls", "log", "num_cpus", @@ -7959,9 +7959,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "structopt" -version = "0.3.14" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "863246aaf5ddd0d6928dfeb1a9ca65f505599e4e1b399935ef7e75107516b4ef" +checksum = "ff6da2e8d107dfd7b74df5ef4d205c6aebee0706c647f6bc6a2d5789905c00fb" dependencies = [ "clap", "lazy_static", @@ -7970,9 +7970,9 @@ dependencies = [ [[package]] name = "structopt-derive" -version = "0.4.7" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d239ca4b13aee7a2142e6795cbd69e457665ff8037aed33b3effdc430d2f927a" +checksum = "a489c87c08fbaf12e386665109dd13470dcc9c4583ea3e10dd2b4523e5ebd9ac" dependencies = [ "heck", "proc-macro-error", @@ -8122,7 +8122,7 @@ dependencies = [ "async-std", "derive_more", "futures-util", - "hyper 0.13.5", + "hyper 0.13.4", "log", "prometheus", "tokio 0.2.18", @@ -8404,9 +8404,9 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.13.4" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cac193374347e7c263c5f547524f36ff8ec6702d56c8799c8331d26dffe8c1e" +checksum = "5a0338198966bde7feb14b011a33d404a62a6e03b843352c71512a2a002634b7" dependencies = [ "cfg-if", "doc-comment", @@ -8514,11 +8514,12 @@ dependencies = [ [[package]] name = "time" -version = "0.1.43" +version = "0.1.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +checksum = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" dependencies = [ "libc", + "redox_syscall", "winapi 0.3.8", ] @@ -9379,18 +9380,18 @@ dependencies = [ [[package]] name = "wast" -version = "14.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b11c94c63d5365a76ea287f8e6e5b6050233fae4b2423aea2a1e126a385e17" +checksum = "5b20abd8b4a26f7e0d4dd5e357e90a3d555ec190e94472c9b2b27c5b9777f9ae" dependencies = [ "leb128", ] [[package]] name = "wat" -version = "1.0.15" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03db18bc33cff3859c296efbefdcc00763a644539feeadca3415a1cee8a2835d" +checksum = "51a615830ee3e7200b505c441fec09aac2f114deae69df52f215cb828ba112c4" dependencies = [ "wast", ] @@ -9472,9 +9473,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "fa515c5163a99cc82bab70fd3bfdd36d827be85de63737b40fcef2ce084a436e" dependencies = [ "winapi 0.3.8", ] diff --git a/substrate/frame/benchmark/src/lib.rs b/substrate/frame/benchmark/src/lib.rs index 6f7b074a9e..037edc9d26 100644 --- a/substrate/frame/benchmark/src/lib.rs +++ b/substrate/frame/benchmark/src/lib.rs @@ -160,7 +160,7 @@ decl_module! { #[weight = 0] pub fn append_member_list(origin) { let who = ensure_signed(origin)?; - MyMemberList::::append(&[who])?; + MyMemberList::::append(who); } /// Encode a vector of accounts to bytes. diff --git a/substrate/frame/democracy/src/lib.rs b/substrate/frame/democracy/src/lib.rs index ddc3b36f4c..d924ca9666 100644 --- a/substrate/frame/democracy/src/lib.rs +++ b/substrate/frame/democracy/src/lib.rs @@ -168,7 +168,7 @@ use sp_runtime::{ DispatchResult, DispatchError, RuntimeDebug, traits::{Zero, Hash, Dispatchable, Saturating}, }; -use codec::{Ref, Encode, Decode}; +use codec::{Encode, Decode}; use frame_support::{ decl_module, decl_storage, decl_event, decl_error, ensure, Parameter, weights::{Weight, DispatchClass}, @@ -553,8 +553,7 @@ decl_module! { PublicPropCount::put(index + 1); >::insert(index, (value, &[&who][..])); - let new_prop = (index, proposal_hash, who); - >::append_or_put(&[Ref::from(&new_prop)][..]); + >::append((index, proposal_hash, who)); Self::deposit_event(RawEvent::Proposed(index, value)); } diff --git a/substrate/frame/elections/src/lib.rs b/substrate/frame/elections/src/lib.rs index 4e3fa9c75d..a684ce3144 100644 --- a/substrate/frame/elections/src/lib.rs +++ b/substrate/frame/elections/src/lib.rs @@ -886,7 +886,7 @@ impl Module { if set_len + 1 == VOTER_SET_SIZE { NextVoterSet::put(next + 1); } - >::append_or_insert(next, &[Some(who.clone())][..]) + >::append(next, Some(who.clone())); } } diff --git a/substrate/frame/scheduler/src/lib.rs b/substrate/frame/scheduler/src/lib.rs index a18a48da08..975331059b 100644 --- a/substrate/frame/scheduler/src/lib.rs +++ b/substrate/frame/scheduler/src/lib.rs @@ -52,7 +52,7 @@ use frame_support::{ traits::{Get, schedule}, weights::{GetDispatchInfo, Weight}, }; -use frame_system::{self as system}; +use frame_system as system; /// Our pallet's configuration trait. All our types and constants go in here. If the /// pallet is dependent on specific other pallets, then their configuration traits @@ -123,7 +123,7 @@ decl_module! { .collect::>(); queued.sort_by_key(|(_, s)| s.priority); let mut result = 0; - let unused_items = queued.into_iter() + queued.into_iter() .enumerate() .scan(0, |cumulative_weight, (order, (index, s))| { *cumulative_weight += s.call.get_dispatch_info().weight; @@ -141,10 +141,10 @@ decl_module! { } let next = now + period; if let Some(ref id) = s.maybe_id { - let next_index = Agenda::::decode_len(now + period).unwrap_or(0) as u32; - Lookup::::insert(id, (next, next_index)); + let next_index = Agenda::::decode_len(now + period).unwrap_or(0); + Lookup::::insert(id, (next, next_index as u32)); } - Agenda::::append_or_insert(next, &[Some(s)][..]); + Agenda::::append(next, Some(s)); } else { if let Some(ref id) = s.maybe_id { Lookup::::remove(id); @@ -161,11 +161,11 @@ decl_module! { Some(Some(s)) } }) - .collect::>(); - if !unused_items.is_empty() { - let next = now + One::one(); - Agenda::::append_or_insert(next, &unused_items[..]); - } + .for_each(|unused| { + let next = now + One::one(); + Agenda::::append(next, unused); + }); + result } } @@ -186,7 +186,7 @@ impl schedule::Anon::Call> for Module // Remove one from the number of repetitions since we will schedule one now. .map(|(p, c)| (p, c - 1)); let s = Some(Scheduled { maybe_id: None, priority, call, maybe_periodic }); - Agenda::::append_or_insert(when, &[s][..]); + Agenda::::append(when, s); (when, Agenda::::decode_len(when).unwrap_or(1) as u32 - 1) } @@ -225,7 +225,7 @@ impl schedule::Named::Call> for Module .map(|(p, c)| (p, c - 1)); let s = Scheduled { maybe_id: Some(id.clone()), priority, call, maybe_periodic }; - Agenda::::append_or_insert(when, &[Some(s)][..]); + Agenda::::append(when, Some(s)); let index = Agenda::::decode_len(when).unwrap_or(1) as u32 - 1; let address = (when, index); Lookup::::insert(&id, &address); diff --git a/substrate/frame/scored-pool/src/lib.rs b/substrate/frame/scored-pool/src/lib.rs index 46eb73293b..1ba999cc7c 100644 --- a/substrate/frame/scored-pool/src/lib.rs +++ b/substrate/frame/scored-pool/src/lib.rs @@ -100,9 +100,7 @@ use frame_support::{ weights::Weight, }; use frame_system::{self as system, ensure_root, ensure_signed}; -use sp_runtime::{ - traits::{AtLeast32Bit, MaybeSerializeDeserialize, Zero, StaticLookup}, -}; +use sp_runtime::traits::{AtLeast32Bit, MaybeSerializeDeserialize, Zero, StaticLookup}; type BalanceOf = <>::Currency as Currency<::AccountId>>::Balance; type PoolT = Vec<(::AccountId, Option<>::Score>)>; @@ -276,10 +274,7 @@ decl_module! { // can be inserted as last element in pool, since entities with // `None` are always sorted to the end. - if let Err(e) = >::append(&[(who.clone(), None)]) { - T::Currency::unreserve(&who, deposit); - Err(e)? - } + >::append((who.clone(), Option::<>::Score>::None)); >::insert(&who, true); diff --git a/substrate/frame/support/src/lib.rs b/substrate/frame/support/src/lib.rs index b0e7395e9f..5222e506d5 100644 --- a/substrate/frame/support/src/lib.rs +++ b/substrate/frame/support/src/lib.rs @@ -499,8 +499,8 @@ mod tests { let key2 = 18u32; DoubleMap::insert(&key1, &key2, &vec![1]); - DoubleMap::append(&key1, &key2, &[2, 3]).unwrap(); - assert_eq!(DoubleMap::get(&key1, &key2), &[1, 2, 3]); + DoubleMap::append(&key1, &key2, 2); + assert_eq!(DoubleMap::get(&key1, &key2), &[1, 2]); }); } diff --git a/substrate/frame/support/src/storage/generator/double_map.rs b/substrate/frame/support/src/storage/generator/double_map.rs index c7e4c10017..2c18089d38 100644 --- a/substrate/frame/support/src/storage/generator/double_map.rs +++ b/substrate/frame/support/src/storage/generator/double_map.rs @@ -16,8 +16,8 @@ use sp_std::prelude::*; use sp_std::borrow::Borrow; -use codec::{Ref, FullCodec, FullEncode, Decode, Encode, EncodeLike, EncodeAppend}; -use crate::{storage::{self, unhashed}, traits::Len, Never}; +use codec::{FullCodec, FullEncode, Decode, Encode, EncodeLike}; +use crate::{storage::{self, unhashed, StorageAppend}, traits::Len, Never}; use crate::hash::{StorageHasher, Twox128, ReversibleStorageHasher}; /// Generator for `StorageDoubleMap` used by `decl_storage`. @@ -245,53 +245,19 @@ impl storage::StorageDoubleMap for G where ret } - fn append( + fn append( k1: KArg1, k2: KArg2, - items: Items, - ) -> Result<(), &'static str> where - KArg1: EncodeLike, - KArg2: EncodeLike, - Item: Encode, - EncodeLikeItem: EncodeLike, - V: EncodeAppend, - Items: IntoIterator, - Items::IntoIter: ExactSizeIterator - { - let final_key = Self::storage_double_map_final_key(k1, k2); - - let encoded_value = unhashed::get_raw(&final_key) - .unwrap_or_else(|| { - match G::from_query_to_optional_value(G::from_optional_value_to_query(None)) { - Some(value) => value.encode(), - None => Vec::new(), - } - }); - - let new_val = V::append_or_new( - encoded_value, - items, - ).map_err(|_| "Could not append given item")?; - unhashed::put_raw(&final_key, &new_val); - - Ok(()) - } - - fn append_or_insert( - k1: KArg1, - k2: KArg2, - items: Items, + item: EncodeLikeItem, ) where KArg1: EncodeLike, KArg2: EncodeLike, Item: Encode, EncodeLikeItem: EncodeLike, - V: EncodeAppend, - Items: IntoIterator + Clone + EncodeLike, - Items::IntoIter: ExactSizeIterator + V: StorageAppend, { - Self::append(Ref::from(&k1), Ref::from(&k2), items.clone()) - .unwrap_or_else(|_| Self::insert(k1, k2, items)); + let final_key = Self::storage_double_map_final_key(k1, k2); + sp_io::storage::append(&final_key, item.encode()); } fn decode_len(key1: KArg1, key2: KArg2) -> Result where diff --git a/substrate/frame/support/src/storage/generator/map.rs b/substrate/frame/support/src/storage/generator/map.rs index cc871072f5..307e3b1c78 100644 --- a/substrate/frame/support/src/storage/generator/map.rs +++ b/substrate/frame/support/src/storage/generator/map.rs @@ -17,8 +17,8 @@ #[cfg(not(feature = "std"))] use sp_std::prelude::*; use sp_std::borrow::Borrow; -use codec::{FullCodec, FullEncode, Decode, Encode, EncodeLike, Ref, EncodeAppend}; -use crate::{storage::{self, unhashed}, traits::Len, Never}; +use codec::{FullCodec, FullEncode, Decode, Encode, EncodeLike}; +use crate::{storage::{self, unhashed, StorageAppend}, traits::Len, Never}; use crate::hash::{StorageHasher, Twox128, ReversibleStorageHasher}; /// Generator for `StorageMap` used by `decl_storage`. @@ -276,43 +276,15 @@ impl> storage::StorageMap G::from_optional_value_to_query(value) } - fn append(key: KeyArg, items: Items) -> Result<(), &'static str> + fn append(key: EncodeLikeKey, item: EncodeLikeItem) where - KeyArg: EncodeLike, + EncodeLikeKey: EncodeLike, Item: Encode, EncodeLikeItem: EncodeLike, - V: EncodeAppend, - Items: IntoIterator, - Items::IntoIter: ExactSizeIterator, + V: StorageAppend, { let key = Self::storage_map_final_key(key); - let encoded_value = unhashed::get_raw(key.as_ref()) - .unwrap_or_else(|| { - match G::from_query_to_optional_value(G::from_optional_value_to_query(None)) { - Some(value) => value.encode(), - None => Vec::new(), - } - }); - - let new_val = V::append_or_new( - encoded_value, - items, - ).map_err(|_| "Could not append given item")?; - unhashed::put_raw(key.as_ref(), &new_val); - Ok(()) - } - - fn append_or_insert(key: KeyArg, items: Items) - where - KeyArg: EncodeLike, - Item: Encode, - EncodeLikeItem: EncodeLike, - V: EncodeAppend, - Items: IntoIterator + Clone + EncodeLike, - Items::IntoIter: ExactSizeIterator, - { - Self::append(Ref::from(&key), items.clone()) - .unwrap_or_else(|_| Self::insert(key, items)); + sp_io::storage::append(&key, item.encode()); } fn decode_len>(key: KeyArg) -> Result diff --git a/substrate/frame/support/src/storage/generator/value.rs b/substrate/frame/support/src/storage/generator/value.rs index dd9bbded4b..f4119ba461 100644 --- a/substrate/frame/support/src/storage/generator/value.rs +++ b/substrate/frame/support/src/storage/generator/value.rs @@ -14,12 +14,10 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -#[cfg(not(feature = "std"))] -use sp_std::prelude::*; -use codec::{FullCodec, Encode, EncodeAppend, EncodeLike, Decode}; +use codec::{FullCodec, Encode, EncodeLike, Decode}; use crate::{ Never, - storage::{self, unhashed}, + storage::{self, unhashed, StorageAppend}, hash::{Twox128, StorageHasher}, traits::Len }; @@ -134,55 +132,16 @@ impl> storage::StorageValue for G { G::from_optional_value_to_query(value) } - /// Append the given items to the value in the storage. - /// - /// `T` is required to implement `codec::EncodeAppend`. - fn append(items: Items) -> Result<(), &'static str> + fn append(item: EncodeLikeItem) where Item: Encode, EncodeLikeItem: EncodeLike, - T: EncodeAppend, - Items: IntoIterator, - Items::IntoIter: ExactSizeIterator, + T: StorageAppend, { let key = Self::storage_value_final_key(); - let encoded_value = unhashed::get_raw(&key) - .unwrap_or_else(|| { - match G::from_query_to_optional_value(G::from_optional_value_to_query(None)) { - Some(value) => value.encode(), - None => Vec::new(), - } - }); - - let new_val = T::append_or_new( - encoded_value, - items, - ).map_err(|_| "Could not append given item")?; - unhashed::put_raw(&key, &new_val); - Ok(()) + sp_io::storage::append(&key, item.encode()); } - /// Safely append the given items to the value in the storage. If a codec error occurs, then the - /// old (presumably corrupt) value is replaced with the given `items`. - /// - /// `T` is required to implement `codec::EncodeAppend`. - fn append_or_put(items: Items) where - Item: Encode, - EncodeLikeItem: EncodeLike, - T: EncodeAppend, - Items: IntoIterator + Clone + EncodeLike, - Items::IntoIter: ExactSizeIterator - { - Self::append(items.clone()).unwrap_or_else(|_| Self::put(items)); - } - - /// Read the length of the value in a fast way, without decoding the entire value. - /// - /// `T` is required to implement `Codec::DecodeLength`. - /// - /// Note that `0` is returned as the default value if no encoded value exists at the given key. - /// Therefore, this function cannot be used as a sign of _existence_. use the `::exists()` - /// function for this purpose. fn decode_len() -> Result where T: codec::DecodeLength, T: Len { let key = Self::storage_value_final_key(); diff --git a/substrate/frame/support/src/storage/mod.rs b/substrate/frame/support/src/storage/mod.rs index 3a811c20e3..902190302d 100644 --- a/substrate/frame/support/src/storage/mod.rs +++ b/substrate/frame/support/src/storage/mod.rs @@ -17,7 +17,7 @@ //! Stuff to do with the runtime's storage. use sp_std::{prelude::*, marker::PhantomData}; -use codec::{FullCodec, FullEncode, Encode, EncodeAppend, EncodeLike, Decode}; +use codec::{FullCodec, FullEncode, Encode, EncodeLike, Decode}; use crate::{traits::Len, hash::{Twox128, StorageHasher}}; pub mod unhashed; @@ -91,33 +91,18 @@ pub trait StorageValue { /// Append the given item to the value in the storage. /// - /// `T` is required to implement `codec::EncodeAppend`. - fn append(items: Items) -> Result<(), &'static str> + /// `T` is required to implement [`StorageAppend`]. + /// + /// # Warning + /// + /// If the storage item is not encoded properly, the storage item will be overwritten + /// and set to `[item]`. Any default value set for the storage item will be ignored + /// on overwrite. + fn append(item: EncodeLikeItem) where Item: Encode, EncodeLikeItem: EncodeLike, - T: EncodeAppend, - Items: IntoIterator, - Items::IntoIter: ExactSizeIterator; - - /// Append the given items to the value in the storage. - /// - /// `T` is required to implement `Codec::EncodeAppend`. - /// - /// Upon any failure, it replaces `items` as the new value (assuming that the previous stored - /// data is simply corrupt and no longer usable). - /// - /// ### WARNING - /// - /// use with care; if your use-case is not _exactly_ as what this function is doing, - /// you should use append and sensibly handle failure within the runtime code if it happens. - fn append_or_put(items: Items) where - Item: Encode, - EncodeLikeItem: EncodeLike, - T: EncodeAppend, - Items: IntoIterator + Clone + EncodeLike, - Items::IntoIter: ExactSizeIterator; - + T: StorageAppend; /// Read the length of the value in a fast way, without decoding the entire value. /// @@ -176,25 +161,18 @@ pub trait StorageMap { /// Append the given items to the value in the storage. /// /// `V` is required to implement `codec::EncodeAppend`. - fn append(key: KeyArg, items: Items) -> Result<(), &'static str> where - KeyArg: EncodeLike, - Item: Encode, - EncodeLikeItem: EncodeLike, - V: EncodeAppend, - Items: IntoIterator, - Items::IntoIter: ExactSizeIterator; - - /// Safely append the given items to the value in the storage. If a codec error occurs, then the - /// old (presumably corrupt) value is replaced with the given `items`. /// - /// `V` is required to implement `codec::EncodeAppend`. - fn append_or_insert(key: KeyArg, items: Items) where - KeyArg: EncodeLike, + /// # Warning + /// + /// If the storage item is not encoded properly, the storage will be overwritten + /// and set to `[item]`. Any default value set for the storage item will be ignored + /// on overwrite. + fn append(key: EncodeLikeKey, item: EncodeLikeItem) + where + EncodeLikeKey: EncodeLike, Item: Encode, EncodeLikeItem: EncodeLike, - V: EncodeAppend, - Items: IntoIterator + Clone + EncodeLike, - Items::IntoIter: ExactSizeIterator; + V: StorageAppend; /// Read the length of the value in a fast way, without decoding the entire value. /// @@ -351,38 +329,23 @@ pub trait StorageDoubleMap { /// Append the given item to the value in the storage. /// - /// `V` is required to implement `codec::EncodeAppend`. - fn append( - k1: KArg1, - k2: KArg2, - items: Items, - ) -> Result<(), &'static str> - where - KArg1: EncodeLike, - KArg2: EncodeLike, - Item: Encode, - EncodeLikeItem: EncodeLike, - V: EncodeAppend, - Items: IntoIterator, - Items::IntoIter: ExactSizeIterator; - - /// Safely append the given items to the value in the storage. If a codec error occurs, then the - /// old (presumably corrupt) value is replaced with the given `items`. + /// `V` is required to implement [`StorageAppend`]. /// - /// `V` is required to implement `codec::EncodeAppend`. - fn append_or_insert( + /// # Warning + /// + /// If the storage item is not encoded properly, the storage will be overwritten + /// and set to `[item]`. Any default value set for the storage item will be ignored + /// on overwrite. + fn append( k1: KArg1, k2: KArg2, - items: Items, - ) - where + item: EncodeLikeItem, + ) where KArg1: EncodeLike, KArg2: EncodeLike, Item: Encode, EncodeLikeItem: EncodeLike, - V: EncodeAppend, - Items: IntoIterator + Clone + EncodeLike, - Items::IntoIter: ExactSizeIterator; + V: StorageAppend; /// Read the length of the value in a fast way, without decoding the entire value. /// @@ -449,7 +412,6 @@ impl Iterator for PrefixIterator { /// Twox128(module_prefix) ++ Twox128(storage_prefix) /// ``` pub trait StoragePrefixedMap { - /// Module prefix. Used for generating final key. fn module_prefix() -> &'static [u8]; @@ -525,6 +487,22 @@ pub trait StoragePrefixedMap { } } +/// Marker trait that will be implemented for types that support the `storage::append` api. +/// +/// This trait is sealed. +pub trait StorageAppend: private::Sealed {} + +/// Provides `Sealed` trait to prevent implementing trait `StorageAppend` outside of this crate. +mod private { + use super::*; + + pub trait Sealed {} + + impl Sealed for Vec {} +} + +impl StorageAppend for Vec {} + #[cfg(test)] mod test { use sp_core::hashing::twox_128; diff --git a/substrate/frame/support/test/tests/decl_storage.rs b/substrate/frame/support/test/tests/decl_storage.rs index ea9b09f9d7..77e538625a 100644 --- a/substrate/frame/support/test/tests/decl_storage.rs +++ b/substrate/frame/support/test/tests/decl_storage.rs @@ -526,50 +526,58 @@ mod test_append_and_len { #[test] fn append_works() { TestExternalities::default().execute_with(|| { - let _ = MapVec::append(1, [1, 2, 3].iter()); - let _ = MapVec::append(1, [4, 5].iter()); + for val in &[1, 2, 3, 4, 5] { + MapVec::append(1, val); + } assert_eq!(MapVec::get(1), vec![1, 2, 3, 4, 5]); - let _ = JustVec::append([1, 2, 3].iter()); - let _ = JustVec::append([4, 5].iter()); + MapVec::remove(1); + MapVec::append(1, 1); + assert_eq!(MapVec::get(1), vec![1]); + + for val in &[1, 2, 3, 4, 5] { + JustVec::append(val); + } assert_eq!(JustVec::get(), vec![1, 2, 3, 4, 5]); + + JustVec::kill(); + JustVec::append(1); + assert_eq!(JustVec::get(), vec![1]); }); } #[test] - fn append_works_for_default() { + fn append_overwrites_invalid_data() { + TestExternalities::default().execute_with(|| { + let key = JustVec::hashed_key(); + // Set it to some invalid value. + frame_support::storage::unhashed::put_raw(&key, &*b"1"); + assert_eq!(JustVec::get(), Vec::new()); + assert_eq!(frame_support::storage::unhashed::get_raw(&key), Some(b"1".to_vec())); + + JustVec::append(1); + JustVec::append(2); + assert_eq!(JustVec::get(), vec![1, 2]); + }); + } + + #[test] + fn append_overwrites_default() { TestExternalities::default().execute_with(|| { assert_eq!(JustVecWithDefault::get(), vec![6, 9]); - let _ = JustVecWithDefault::append([1].iter()); - assert_eq!(JustVecWithDefault::get(), vec![6, 9, 1]); + JustVecWithDefault::append(1); + assert_eq!(JustVecWithDefault::get(), vec![1]); assert_eq!(MapVecWithDefault::get(0), vec![6, 9]); - let _ = MapVecWithDefault::append(0, [1].iter()); - assert_eq!(MapVecWithDefault::get(0), vec![6, 9, 1]); + MapVecWithDefault::append(0, 1); + assert_eq!(MapVecWithDefault::get(0), vec![1]); assert_eq!(OptionVec::get(), None); - let _ = OptionVec::append([1].iter()); + OptionVec::append(1); assert_eq!(OptionVec::get(), Some(vec![1])); }); } - #[test] - fn append_or_put_works() { - TestExternalities::default().execute_with(|| { - let _ = MapVec::append_or_insert(1, &[1, 2, 3][..]); - let _ = MapVec::append_or_insert(1, &[4, 5][..]); - assert_eq!(MapVec::get(1), vec![1, 2, 3, 4, 5]); - - let _ = JustVec::append_or_put(&[1, 2, 3][..]); - let _ = JustVec::append_or_put(&[4, 5][..]); - assert_eq!(JustVec::get(), vec![1, 2, 3, 4, 5]); - - let _ = OptionVec::append_or_put(&[1, 2, 3][..]); - let _ = OptionVec::append_or_put(&[4, 5][..]); - assert_eq!(OptionVec::get(), Some(vec![1, 2, 3, 4, 5])); - }); - } - #[test] fn len_works() { TestExternalities::default().execute_with(|| { diff --git a/substrate/frame/system/src/lib.rs b/substrate/frame/system/src/lib.rs index 5a3097e62e..50778e7222 100644 --- a/substrate/frame/system/src/lib.rs +++ b/substrate/frame/system/src/lib.rs @@ -116,7 +116,7 @@ use sp_runtime::{ use sp_core::{ChangesTrieConfiguration, storage::well_known_keys}; use frame_support::{ decl_module, decl_event, decl_storage, decl_error, Parameter, ensure, debug, - storage::{self, generator::StorageValue}, + storage, traits::{ Contains, Get, ModuleToIndex, OnNewAccount, OnKilledAccount, IsDeadAccount, Happened, StoredMap, EnsureOrigin, @@ -852,16 +852,10 @@ impl Module { old_event_count }; - // We use append api here to avoid bringing all events in the runtime when we push a - // new one in the list. - let encoded_event = event.encode(); - sp_io::storage::append(&Events::::storage_value_final_key()[..], encoded_event); + Events::::append(&event); for topic in topics { - // The same applies here. - if >::append(topic, &[(block_number, event_idx)]).is_err() { - return; - } + >::append(topic, &(block_number, event_idx)); } } diff --git a/substrate/primitives/core/src/testing.rs b/substrate/primitives/core/src/testing.rs index c223811124..f818865d21 100644 --- a/substrate/primitives/core/src/testing.rs +++ b/substrate/primitives/core/src/testing.rs @@ -25,7 +25,6 @@ use crate::{ }; #[cfg(feature = "std")] use std::collections::HashSet; -use codec::Encode; /// Key type for generic Ed25519 key. pub const ED25519: KeyTypeId = KeyTypeId(*b"ed25"); /// Key type for generic Sr 25519 key. @@ -172,6 +171,8 @@ impl crate::traits::BareCryptoStore for KeyStore { key: &CryptoTypePublicPair, msg: &[u8], ) -> Result, BareCryptoStoreError> { + use codec::Encode; + match key.0 { ed25519::CRYPTO_ID => { let key_pair: ed25519::Pair = self diff --git a/substrate/primitives/io/src/lib.rs b/substrate/primitives/io/src/lib.rs index e4787dade0..f5d692469f 100644 --- a/substrate/primitives/io/src/lib.rs +++ b/substrate/primitives/io/src/lib.rs @@ -117,7 +117,14 @@ pub trait Storage { Externalities::clear_prefix(*self, prefix) } - /// Append to storage item (assumes it is in "Vec" format). + /// Append the encoded `value` to the storage item at `key`. + /// + /// The storage item needs to implement [`EncodeAppend`](codec::EncodeAppend). + /// + /// # Warning + /// + /// If the storage item does not support [`EncodeAppend`](codec::EncodeAppend) or + /// something else fails at appending, the storage item will be set to `[value]`. fn append(&mut self, key: &[u8], value: Vec) { self.storage_append(key.to_vec(), value); } diff --git a/substrate/primitives/state-machine/src/basic.rs b/substrate/primitives/state-machine/src/basic.rs index 4eb724ce68..450b1b316e 100644 --- a/substrate/primitives/state-machine/src/basic.rs +++ b/substrate/primitives/state-machine/src/basic.rs @@ -262,8 +262,8 @@ impl Externalities for BasicExternalities { key: Vec, value: Vec, ) { - let previous = self.inner.top.entry(key).or_default(); - crate::ext::append_to_storage(previous, value).expect("Failed to append to storage"); + let current = self.inner.top.entry(key).or_default(); + crate::ext::StorageAppend::new(current).append(value); } fn chain_id(&self) -> u64 { 42 } diff --git a/substrate/primitives/state-machine/src/ext.rs b/substrate/primitives/state-machine/src/ext.rs index 902763e6e9..746e016a64 100644 --- a/substrate/primitives/state-machine/src/ext.rs +++ b/substrate/primitives/state-machine/src/ext.rs @@ -30,7 +30,7 @@ use sp_core::{ }; use sp_trie::{trie_types::Layout, empty_child_trie_root}; use sp_externalities::{Extensions, Extension}; -use codec::{Compact, Decode, Encode}; +use codec::{Decode, Encode, EncodeAppend}; use std::{error, fmt, any::{Any, TypeId}}; use log::{warn, trace}; @@ -428,7 +428,7 @@ where &key, || backend.storage(&key).expect(EXT_NOT_ALLOWED_TO_FAIL).unwrap_or_default() ); - append_to_storage(current_value, value).expect(EXT_NOT_ALLOWED_TO_FAIL); + StorageAppend::new(current_value).append(value); } fn chain_id(&self) -> u64 { @@ -549,16 +549,24 @@ where fn wipe(&mut self) { self.overlay.discard_prospective(); - self.overlay.drain_storage_changes(&self.backend, None, Default::default(), self.storage_transaction_cache) - .expect(EXT_NOT_ALLOWED_TO_FAIL); + self.overlay.drain_storage_changes( + &self.backend, + None, + Default::default(), + self.storage_transaction_cache, + ).expect(EXT_NOT_ALLOWED_TO_FAIL); self.storage_transaction_cache.reset(); self.backend.wipe().expect(EXT_NOT_ALLOWED_TO_FAIL) } fn commit(&mut self) { self.overlay.commit_prospective(); - let changes = self.overlay.drain_storage_changes(&self.backend, None, Default::default(), self.storage_transaction_cache) - .expect(EXT_NOT_ALLOWED_TO_FAIL); + let changes = self.overlay.drain_storage_changes( + &self.backend, + None, + Default::default(), + self.storage_transaction_cache, + ).expect(EXT_NOT_ALLOWED_TO_FAIL); self.backend.commit( changes.transaction_storage_root, changes.transaction, @@ -567,65 +575,43 @@ where } } -fn extract_length_data(data: &[u8]) -> Result<(u32, usize, usize), &'static str> { - use codec::CompactLen; - let len = u32::from( - Compact::::decode(&mut &data[..]) - .map_err(|_| "Incorrect updated item encoding")? - ); - let new_len = len - .checked_add(1) - .ok_or_else(|| "New vec length greater than `u32::max_value()`.")?; +/// Implement `Encode` by forwarding the stored raw vec. +struct EncodeOpaqueValue(Vec); - let encoded_len = Compact::::compact_len(&len); - let encoded_new_len = Compact::::compact_len(&new_len); - - Ok((new_len, encoded_len, encoded_new_len)) +impl Encode for EncodeOpaqueValue { + fn using_encoded R>(&self, f: F) -> R { + f(&self.0) + } } -pub fn append_to_storage( - self_encoded: &mut Vec, - value: Vec, -) -> Result<(), &'static str> { - // No data present, just encode the given input data. - if self_encoded.is_empty() { - Compact::from(1u32).encode_to(self_encoded); - self_encoded.extend(value); - return Ok(()); +/// Auxialiary structure for appending a value to a storage item. +pub(crate) struct StorageAppend<'a>(&'a mut Vec); + +impl<'a> StorageAppend<'a> { + /// Create a new instance using the given `storage` reference. + pub fn new(storage: &'a mut Vec) -> Self { + Self(storage) } - let (new_len, encoded_len, encoded_new_len) = extract_length_data(&self_encoded)?; + /// Append the given `value` to the storage item. + /// + /// If appending fails, `[value]` is stored in the storage item. + pub fn append(&mut self, value: Vec) { + let value = vec![EncodeOpaqueValue(value)]; - let replace_len = |dest: &mut Vec| { - Compact(new_len).using_encoded(|e| { - dest[..encoded_new_len].copy_from_slice(e); - }) - }; + let item = std::mem::take(self.0); - let append_new_elems = |dest: &mut Vec| dest.extend(&value[..]); - - // If old and new encoded len is equal, we don't need to copy the - // already encoded data. - if encoded_len == encoded_new_len { - replace_len(self_encoded); - append_new_elems(self_encoded); - - Ok(()) - } else { - let size = encoded_new_len + self_encoded.len() - encoded_len; - - let mut res = Vec::with_capacity(size + value.len()); - unsafe { res.set_len(size); } - - // Insert the new encoded len, copy the already encoded data and - // add the new element. - replace_len(&mut res); - res[encoded_new_len..size].copy_from_slice(&self_encoded[encoded_len..]); - append_new_elems(&mut res); - *self_encoded = res; - - Ok(()) + *self.0 = match Vec::::append_or_new(item, &value) { + Ok(item) => item, + Err(_) => { + log::error!( + target: "runtime", + "Failed to append value, resetting storage item to `[value]`.", + ); + value.encode() + } + }; } } @@ -902,4 +888,24 @@ mod tests { Some(Blake2Hasher::hash(&[31]).as_ref().to_vec()), ); } + + #[test] + fn storage_append_works() { + let mut data = Vec::new(); + let mut append = StorageAppend::new(&mut data); + append.append(1u32.encode()); + append.append(2u32.encode()); + drop(append); + + assert_eq!(Vec::::decode(&mut &data[..]).unwrap(), vec![1, 2]); + + // Initialize with some invalid data + let mut data = vec![1]; + let mut append = StorageAppend::new(&mut data); + append.append(1u32.encode()); + append.append(2u32.encode()); + drop(append); + + assert_eq!(Vec::::decode(&mut &data[..]).unwrap(), vec![1, 2]); + } } diff --git a/substrate/primitives/storage/src/lib.rs b/substrate/primitives/storage/src/lib.rs index ed90c4584b..c3e8d35075 100644 --- a/substrate/primitives/storage/src/lib.rs +++ b/substrate/primitives/storage/src/lib.rs @@ -22,8 +22,7 @@ use serde::{Serialize, Deserialize}; use sp_debug_derive::RuntimeDebug; -use sp_std::vec::Vec; -use sp_std::ops::{Deref, DerefMut}; +use sp_std::{vec::Vec, ops::{Deref, DerefMut}}; use ref_cast::RefCast; /// Storage key. @@ -334,10 +333,15 @@ impl ChildTrieParentKeyId { } } -#[test] -fn test_prefix_default_child_info() { - let child_info = ChildInfo::new_default(b"any key"); - let prefix = child_info.child_type().parent_prefix(); - assert!(prefix.starts_with(well_known_keys::CHILD_STORAGE_KEY_PREFIX)); - assert!(prefix.starts_with(well_known_keys::DEFAULT_CHILD_STORAGE_KEY_PREFIX)); +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_prefix_default_child_info() { + let child_info = ChildInfo::new_default(b"any key"); + let prefix = child_info.child_type().parent_prefix(); + assert!(prefix.starts_with(well_known_keys::CHILD_STORAGE_KEY_PREFIX)); + assert!(prefix.starts_with(well_known_keys::DEFAULT_CHILD_STORAGE_KEY_PREFIX)); + } } diff --git a/substrate/primitives/wasm-interface/src/lib.rs b/substrate/primitives/wasm-interface/src/lib.rs index eda2ebb1b5..1601590e90 100644 --- a/substrate/primitives/wasm-interface/src/lib.rs +++ b/substrate/primitives/wasm-interface/src/lib.rs @@ -119,7 +119,7 @@ mod private { /// Something that can be wrapped in a wasm `Pointer`. /// /// This trait is sealed. -pub trait PointerType: Sized { +pub trait PointerType: Sized + private::Sealed { /// The size of the type in wasm. const SIZE: u32 = mem::size_of::() as u32; }