Use storage::append in the implementation of the storage types (#5889)

* Start improving `storage_append`

* Fix some stuff

* Fix compilation

* Update docs and add new test

* More tests

* Test kill + append

substrate/Cargo.lock

@@ -126,15 +126,15 @@ dependencies = [
 
 [[package]]
 name = "arbitrary"
-version = "0.4.2"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1148c9b25d393a07c4cc3ef5dd30f82a40a1c261018c4a670611ed8e76cad3ea"
+checksum = "75153c95fdedd7db9732dfbfc3702324a1627eec91ba56e37cd0ac78314ab2ed"
 
 [[package]]
 name = "arc-swap"
-version = "0.4.6"
+version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b585a98a234c46fc563103e9278c9391fde1f4e6850334da895d27edb9580f62"
+checksum = "d663a8e9a99154b5fb793032533f6328da35e23aac63d5c152279aa8ba356825"
 
 [[package]]
 name = "arrayref"
@@ -280,9 +280,9 @@ dependencies = [
 
 [[package]]
 name = "backtrace-sys"
-version = "0.1.36"
+version = "0.1.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78848718ee1255a2485d1309ad9cdecfc2e7d0362dd11c6829364c6b35ae1bc7"
+checksum = "7de8aba10a69c8e8d7622c5710229485ec32e9d55fdad160ea559c086fdcd118"
 dependencies = [
  "cc",
  "libc",
@@ -452,9 +452,9 @@ dependencies = [
 
 [[package]]
 name = "bs58"
-version = "0.3.1"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "476e9cd489f9e121e02ffa6014a8ef220ecb15c05ed23fc34cca13925dc283fb"
+checksum = "b170cd256a3f9fa6b9edae3e44a7dfdfc77e8124dbc3e2612d75f9c3e2396dae"
 
 [[package]]
 name = "bstr"
@@ -553,9 +553,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.0.51"
+version = "1.0.50"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c9384ca4b90c0ea47e19a5c996d6643a3e73dedf9b89c65efb67587e34da1bb"
+checksum = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd"
 dependencies = [
  "jobserver",
 ]
@@ -1147,18 +1147,18 @@ dependencies = [
 
 [[package]]
 name = "enumflags2"
-version = "0.6.4"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83c8d82922337cd23a15f88b70d8e4ef5f11da38dd7cdb55e84dd5de99695da0"
+checksum = "a80e524ebf194285b57e5e7944018721c7fffc673253f5183f7accd88a2a3b0c"
 dependencies = [
  "enumflags2_derive",
 ]
 
 [[package]]
 name = "enumflags2_derive"
-version = "0.6.4"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "946ee94e3dbf58fdd324f9ce245c7b238d46a66f00e86a020b71996349e46cce"
+checksum = "2ed9afacaea0301eefb738c9deea725e6d53938004597cdc518a8cf9a7aa2f03"
 dependencies = [
  "proc-macro2",
  "quote 1.0.3",
@@ -2001,9 +2001,9 @@ dependencies = [
 
 [[package]]
 name = "hermit-abi"
-version = "0.1.11"
+version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a0d737e0f947a1864e93d33fdef4af8445a00d1ed8dc0c8ddb73139ea6abf15"
+checksum = "725cf19794cf90aa94e65050cb4191ff5d8fa87a498383774c47b332e3af952e"
 dependencies = [
  "libc",
 ]
@@ -2156,9 +2156,9 @@ dependencies = [
 
 [[package]]
 name = "hyper"
-version = "0.13.5"
+version = "0.13.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96816e1d921eca64d208a85aab4f7798455a8e34229ee5a88c935bdee1b78b14"
+checksum = "ed6081100e960d9d74734659ffc9cc91daf1c0fc7aceb8eaa94ee1a3f5046f2e"
 dependencies = [
  "bytes 0.5.4",
  "futures-channel",
@@ -2187,7 +2187,7 @@ dependencies = [
  "bytes 0.5.4",
  "ct-logs",
  "futures-util",
- "hyper 0.13.5",
+ "hyper 0.13.4",
  "log",
  "rustls",
  "rustls-native-certs",
@@ -3857,9 +3857,9 @@ dependencies = [
 
 [[package]]
 name = "num_cpus"
-version = "1.13.0"
+version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
+checksum = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6"
 dependencies = [
  "hermit-abi",
  "libc",
@@ -4880,7 +4880,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e"
 dependencies = [
  "lock_api",
- "parking_lot_core 0.7.2",
+ "parking_lot_core 0.7.1",
 ]
 
 [[package]]
@@ -4900,9 +4900,9 @@ dependencies = [
 
 [[package]]
 name = "parking_lot_core"
-version = "0.7.2"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3"
+checksum = "0e136c1904604defe99ce5fd71a28d473fa60a12255d511aa78a9ddf11237aeb"
 dependencies = [
  "cfg-if",
  "cloudabi",
@@ -5604,9 +5604,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.3.7"
+version = "1.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692"
+checksum = "7f6946991529684867e47d86474e3a6d0c0ab9b82d5821e314b1ede31fa3a4b3"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -6545,7 +6545,7 @@ dependencies = [
  "fnv",
  "futures 0.3.4",
  "futures-timer 3.0.2",
- "hyper 0.13.5",
+ "hyper 0.13.4",
  "hyper-rustls",
  "log",
  "num_cpus",
@@ -7959,9 +7959,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
 
 [[package]]
 name = "structopt"
-version = "0.3.14"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "863246aaf5ddd0d6928dfeb1a9ca65f505599e4e1b399935ef7e75107516b4ef"
+checksum = "ff6da2e8d107dfd7b74df5ef4d205c6aebee0706c647f6bc6a2d5789905c00fb"
 dependencies = [
  "clap",
  "lazy_static",
@@ -7970,9 +7970,9 @@ dependencies = [
 
 [[package]]
 name = "structopt-derive"
-version = "0.4.7"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d239ca4b13aee7a2142e6795cbd69e457665ff8037aed33b3effdc430d2f927a"
+checksum = "a489c87c08fbaf12e386665109dd13470dcc9c4583ea3e10dd2b4523e5ebd9ac"
 dependencies = [
  "heck",
  "proc-macro-error",
@@ -8122,7 +8122,7 @@ dependencies = [
  "async-std",
  "derive_more",
  "futures-util",
- "hyper 0.13.5",
+ "hyper 0.13.4",
  "log",
  "prometheus",
  "tokio 0.2.18",
@@ -8404,9 +8404,9 @@ dependencies = [
 
 [[package]]
 name = "sysinfo"
-version = "0.13.4"
+version = "0.13.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cac193374347e7c263c5f547524f36ff8ec6702d56c8799c8331d26dffe8c1e"
+checksum = "5a0338198966bde7feb14b011a33d404a62a6e03b843352c71512a2a002634b7"
 dependencies = [
  "cfg-if",
  "doc-comment",
@@ -8514,11 +8514,12 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.1.43"
+version = "0.1.42"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438"
+checksum = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f"
 dependencies = [
  "libc",
+ "redox_syscall",
  "winapi 0.3.8",
 ]
 
@@ -9379,18 +9380,18 @@ dependencies = [
 
 [[package]]
 name = "wast"
-version = "14.0.0"
+version = "13.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47b11c94c63d5365a76ea287f8e6e5b6050233fae4b2423aea2a1e126a385e17"
+checksum = "5b20abd8b4a26f7e0d4dd5e357e90a3d555ec190e94472c9b2b27c5b9777f9ae"
 dependencies = [
  "leb128",
 ]
 
 [[package]]
 name = "wat"
-version = "1.0.15"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03db18bc33cff3859c296efbefdcc00763a644539feeadca3415a1cee8a2835d"
+checksum = "51a615830ee3e7200b505c441fec09aac2f114deae69df52f215cb828ba112c4"
 dependencies = [
  "wast",
 ]
@@ -9472,9 +9473,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
 [[package]]
 name = "winapi-util"
-version = "0.1.5"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+checksum = "fa515c5163a99cc82bab70fd3bfdd36d827be85de63737b40fcef2ce084a436e"
 dependencies = [
  "winapi 0.3.8",
 ]

substrate/frame/benchmark/src/lib.rs

@@ -160,7 +160,7 @@ decl_module! {
 		#[weight = 0]
 		pub fn append_member_list(origin) {
 			let who = ensure_signed(origin)?;
-			MyMemberList::<T>::append(&[who])?;
+			MyMemberList::<T>::append(who);
 		}
 
 		/// Encode a vector of accounts to bytes.

substrate/frame/democracy/src/lib.rs

@@ -168,7 +168,7 @@ use sp_runtime::{
 	DispatchResult, DispatchError, RuntimeDebug,
 	traits::{Zero, Hash, Dispatchable, Saturating},
 };
-use codec::{Ref, Encode, Decode};
+use codec::{Encode, Decode};
 use frame_support::{
 	decl_module, decl_storage, decl_event, decl_error, ensure, Parameter,
 	weights::{Weight, DispatchClass},
@@ -553,8 +553,7 @@ decl_module! {
 			PublicPropCount::put(index + 1);
 			<DepositOf<T>>::insert(index, (value, &[&who][..]));
 
-			let new_prop = (index, proposal_hash, who);
+			<PublicProps<T>>::append((index, proposal_hash, who));
-			<PublicProps<T>>::append_or_put(&[Ref::from(&new_prop)][..]);
 
 			Self::deposit_event(RawEvent::Proposed(index, value));
 		}

substrate/frame/elections/src/lib.rs

@@ -886,7 +886,7 @@ impl<T: Trait> Module<T> {
 					if set_len + 1 == VOTER_SET_SIZE {
 						NextVoterSet::put(next + 1);
 					}
-					<Voters<T>>::append_or_insert(next, &[Some(who.clone())][..])
+					<Voters<T>>::append(next, Some(who.clone()));
 				}
 			}
 

substrate/frame/scheduler/src/lib.rs

@@ -52,7 +52,7 @@ use frame_support::{
 	traits::{Get, schedule},
 	weights::{GetDispatchInfo, Weight},
 };
-use frame_system::{self as system};
+use frame_system as system;
 
 /// Our pallet's configuration trait. All our types and constants go in here. If the
 /// pallet is dependent on specific other pallets, then their configuration traits
@@ -123,7 +123,7 @@ decl_module! {
 				.collect::<Vec<_>>();
 			queued.sort_by_key(|(_, s)| s.priority);
 			let mut result = 0;
-			let unused_items = queued.into_iter()
+			queued.into_iter()
 				.enumerate()
 				.scan(0, |cumulative_weight, (order, (index, s))| {
 					*cumulative_weight += s.call.get_dispatch_info().weight;
@@ -141,10 +141,10 @@ decl_module! {
 							}
 							let next = now + period;
 							if let Some(ref id) = s.maybe_id {
-								let next_index = Agenda::<T>::decode_len(now + period).unwrap_or(0) as u32;
+								let next_index = Agenda::<T>::decode_len(now + period).unwrap_or(0);
-								Lookup::<T>::insert(id, (next, next_index));
+								Lookup::<T>::insert(id, (next, next_index as u32));
 							}
-							Agenda::<T>::append_or_insert(next, &[Some(s)][..]);
+							Agenda::<T>::append(next, Some(s));
 						} else {
 							if let Some(ref id) = s.maybe_id {
 								Lookup::<T>::remove(id);
@@ -161,11 +161,11 @@ decl_module! {
 						Some(Some(s))
 					}
 				})
-				.collect::<Vec<_>>();
+				.for_each(|unused| {
-			if !unused_items.is_empty() {
+					let next = now + One::one();
-				let next = now + One::one();
+					Agenda::<T>::append(next, unused);
-				Agenda::<T>::append_or_insert(next, &unused_items[..]);
+				});
-			}
+
 			result
 		}
 	}
@@ -186,7 +186,7 @@ impl<T: Trait> schedule::Anon<T::BlockNumber, <T as Trait>::Call> for Module<T>
 			// Remove one from the number of repetitions since we will schedule one now.
 			.map(|(p, c)| (p, c - 1));
 		let s = Some(Scheduled { maybe_id: None, priority, call, maybe_periodic });
-		Agenda::<T>::append_or_insert(when, &[s][..]);
+		Agenda::<T>::append(when, s);
 		(when, Agenda::<T>::decode_len(when).unwrap_or(1) as u32 - 1)
 	}
 
@@ -225,7 +225,7 @@ impl<T: Trait> schedule::Named<T::BlockNumber, <T as Trait>::Call> for Module<T>
 			.map(|(p, c)| (p, c - 1));
 
 		let s = Scheduled { maybe_id: Some(id.clone()), priority, call, maybe_periodic };
-		Agenda::<T>::append_or_insert(when, &[Some(s)][..]);
+		Agenda::<T>::append(when, Some(s));
 		let index = Agenda::<T>::decode_len(when).unwrap_or(1) as u32 - 1;
 		let address = (when, index);
 		Lookup::<T>::insert(&id, &address);

substrate/frame/scored-pool/src/lib.rs

@@ -100,9 +100,7 @@ use frame_support::{
 	weights::Weight,
 };
 use frame_system::{self as system, ensure_root, ensure_signed};
-use sp_runtime::{
+use sp_runtime::traits::{AtLeast32Bit, MaybeSerializeDeserialize, Zero, StaticLookup};
-	traits::{AtLeast32Bit, MaybeSerializeDeserialize, Zero, StaticLookup},
-};
 
 type BalanceOf<T, I> = <<T as Trait<I>>::Currency as Currency<<T as frame_system::Trait>::AccountId>>::Balance;
 type PoolT<T, I> = Vec<(<T as frame_system::Trait>::AccountId, Option<<T as Trait<I>>::Score>)>;
@@ -276,10 +274,7 @@ decl_module! {
 
 			// can be inserted as last element in pool, since entities with
 			// `None` are always sorted to the end.
-			if let Err(e) = <Pool<T, I>>::append(&[(who.clone(), None)]) {
+			<Pool<T, I>>::append((who.clone(), Option::<<T as Trait<I>>::Score>::None));
-				T::Currency::unreserve(&who, deposit);
-				Err(e)?
-			}
 
 			<CandidateExists<T, I>>::insert(&who, true);
 

substrate/frame/support/src/lib.rs

@@ -499,8 +499,8 @@ mod tests {
 			let key2 = 18u32;
 
 			DoubleMap::insert(&key1, &key2, &vec![1]);
-			DoubleMap::append(&key1, &key2, &[2, 3]).unwrap();
+			DoubleMap::append(&key1, &key2, 2);
-			assert_eq!(DoubleMap::get(&key1, &key2), &[1, 2, 3]);
+			assert_eq!(DoubleMap::get(&key1, &key2), &[1, 2]);
 		});
 	}
 

substrate/frame/support/src/storage/generator/double_map.rs

@@ -16,8 +16,8 @@
 
 use sp_std::prelude::*;
 use sp_std::borrow::Borrow;
-use codec::{Ref, FullCodec, FullEncode, Decode, Encode, EncodeLike, EncodeAppend};
+use codec::{FullCodec, FullEncode, Decode, Encode, EncodeLike};
-use crate::{storage::{self, unhashed}, traits::Len, Never};
+use crate::{storage::{self, unhashed, StorageAppend}, traits::Len, Never};
 use crate::hash::{StorageHasher, Twox128, ReversibleStorageHasher};
 
 /// Generator for `StorageDoubleMap` used by `decl_storage`.
@@ -245,53 +245,19 @@ impl<K1, K2, V, G> storage::StorageDoubleMap<K1, K2, V> for G where
 		ret
 	}
 
-	fn append<Items, Item, EncodeLikeItem, KArg1, KArg2>(
+	fn append<Item, EncodeLikeItem, KArg1, KArg2>(
 		k1: KArg1,
 		k2: KArg2,
-		items: Items,
+		item: EncodeLikeItem,
-	) -> Result<(), &'static str> where
-		KArg1: EncodeLike<K1>,
-		KArg2: EncodeLike<K2>,
-		Item: Encode,
-		EncodeLikeItem: EncodeLike<Item>,
-		V: EncodeAppend<Item=Item>,
-		Items: IntoIterator<Item=EncodeLikeItem>,
-		Items::IntoIter: ExactSizeIterator
-	{
-		let final_key = Self::storage_double_map_final_key(k1, k2);
-
-		let encoded_value = unhashed::get_raw(&final_key)
-			.unwrap_or_else(|| {
-				match G::from_query_to_optional_value(G::from_optional_value_to_query(None)) {
-					Some(value) => value.encode(),
-					None => Vec::new(),
-				}
-			});
-
-		let new_val = V::append_or_new(
-			encoded_value,
-			items,
-		).map_err(|_| "Could not append given item")?;
-		unhashed::put_raw(&final_key, &new_val);
-
-		Ok(())
-	}
-
-	fn append_or_insert<Items, Item, EncodeLikeItem, KArg1, KArg2>(
-		k1: KArg1,
-		k2: KArg2,
-		items: Items,
 	) where
 		KArg1: EncodeLike<K1>,
 		KArg2: EncodeLike<K2>,
 		Item: Encode,
 		EncodeLikeItem: EncodeLike<Item>,
-		V: EncodeAppend<Item=Item>,
+		V: StorageAppend<Item>,
-		Items: IntoIterator<Item=EncodeLikeItem> + Clone + EncodeLike<V>,
-		Items::IntoIter: ExactSizeIterator
 	{
-		Self::append(Ref::from(&k1), Ref::from(&k2), items.clone())
+		let final_key = Self::storage_double_map_final_key(k1, k2);
-			.unwrap_or_else(|_| Self::insert(k1, k2, items));
+		sp_io::storage::append(&final_key, item.encode());
 	}
 
 	fn decode_len<KArg1, KArg2>(key1: KArg1, key2: KArg2) -> Result<usize, &'static str> where

substrate/frame/support/src/storage/generator/map.rs

@@ -17,8 +17,8 @@
 #[cfg(not(feature = "std"))]
 use sp_std::prelude::*;
 use sp_std::borrow::Borrow;
-use codec::{FullCodec, FullEncode, Decode, Encode, EncodeLike, Ref, EncodeAppend};
+use codec::{FullCodec, FullEncode, Decode, Encode, EncodeLike};
-use crate::{storage::{self, unhashed}, traits::Len, Never};
+use crate::{storage::{self, unhashed, StorageAppend}, traits::Len, Never};
 use crate::hash::{StorageHasher, Twox128, ReversibleStorageHasher};
 
 /// Generator for `StorageMap` used by `decl_storage`.
@@ -276,43 +276,15 @@ impl<K: FullEncode, V: FullCodec, G: StorageMap<K, V>> storage::StorageMap<K, V>
 		G::from_optional_value_to_query(value)
 	}
 
-	fn append<Items, Item, EncodeLikeItem, KeyArg>(key: KeyArg, items: Items) -> Result<(), &'static str>
+	fn append<Item, EncodeLikeItem, EncodeLikeKey>(key: EncodeLikeKey, item: EncodeLikeItem)
 	where
-		KeyArg: EncodeLike<K>,
+		EncodeLikeKey: EncodeLike<K>,
 		Item: Encode,
 		EncodeLikeItem: EncodeLike<Item>,
-		V: EncodeAppend<Item=Item>,
+		V: StorageAppend<Item>,
-		Items: IntoIterator<Item=EncodeLikeItem>,
-		Items::IntoIter: ExactSizeIterator,
 	{
 		let key = Self::storage_map_final_key(key);
-		let encoded_value = unhashed::get_raw(key.as_ref())
+		sp_io::storage::append(&key, item.encode());
-			.unwrap_or_else(|| {
-				match G::from_query_to_optional_value(G::from_optional_value_to_query(None)) {
-					Some(value) => value.encode(),
-					None => Vec::new(),
-				}
-			});
-
-		let new_val = V::append_or_new(
-			encoded_value,
-			items,
-		).map_err(|_| "Could not append given item")?;
-		unhashed::put_raw(key.as_ref(), &new_val);
-		Ok(())
-	}
-
-	fn append_or_insert<Items, Item, EncodeLikeItem, KeyArg>(key: KeyArg, items: Items)
-	where
-		KeyArg: EncodeLike<K>,
-		Item: Encode,
-		EncodeLikeItem: EncodeLike<Item>,
-		V: EncodeAppend<Item=Item>,
-		Items: IntoIterator<Item=EncodeLikeItem> + Clone + EncodeLike<V>,
-		Items::IntoIter: ExactSizeIterator,
-	{
-		Self::append(Ref::from(&key), items.clone())
-			.unwrap_or_else(|_| Self::insert(key, items));
 	}
 
 	fn decode_len<KeyArg: EncodeLike<K>>(key: KeyArg) -> Result<usize, &'static str>

substrate/frame/support/src/storage/generator/value.rs

@@ -14,12 +14,10 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
-#[cfg(not(feature = "std"))]
+use codec::{FullCodec, Encode, EncodeLike, Decode};
-use sp_std::prelude::*;
-use codec::{FullCodec, Encode, EncodeAppend, EncodeLike, Decode};
 use crate::{
 	Never,
-	storage::{self, unhashed},
+	storage::{self, unhashed, StorageAppend},
 	hash::{Twox128, StorageHasher},
 	traits::Len
 };
@@ -134,55 +132,16 @@ impl<T: FullCodec, G: StorageValue<T>> storage::StorageValue<T> for G {
 		G::from_optional_value_to_query(value)
 	}
 
-	/// Append the given items to the value in the storage.
+	fn append<Item, EncodeLikeItem>(item: EncodeLikeItem)
-	///
-	/// `T` is required to implement `codec::EncodeAppend`.
-	fn append<Items, Item, EncodeLikeItem>(items: Items) -> Result<(), &'static str>
 	where
 		Item: Encode,
 		EncodeLikeItem: EncodeLike<Item>,
-		T: EncodeAppend<Item=Item>,
+		T: StorageAppend<Item>,
-		Items: IntoIterator<Item=EncodeLikeItem>,
-		Items::IntoIter: ExactSizeIterator,
 	{
 		let key = Self::storage_value_final_key();
-		let encoded_value = unhashed::get_raw(&key)
+		sp_io::storage::append(&key, item.encode());
-			.unwrap_or_else(|| {
-				match G::from_query_to_optional_value(G::from_optional_value_to_query(None)) {
-					Some(value) => value.encode(),
-					None => Vec::new(),
-				}
-			});
-
-		let new_val = T::append_or_new(
-			encoded_value,
-			items,
-		).map_err(|_| "Could not append given item")?;
-		unhashed::put_raw(&key, &new_val);
-		Ok(())
 	}
 
-	/// Safely append the given items to the value in the storage. If a codec error occurs, then the
-	/// old (presumably corrupt) value is replaced with the given `items`.
-	///
-	/// `T` is required to implement `codec::EncodeAppend`.
-	fn append_or_put<Items, Item, EncodeLikeItem>(items: Items) where
-		Item: Encode,
-		EncodeLikeItem: EncodeLike<Item>,
-		T: EncodeAppend<Item=Item>,
-		Items: IntoIterator<Item=EncodeLikeItem> + Clone + EncodeLike<T>,
-		Items::IntoIter: ExactSizeIterator
-	{
-		Self::append(items.clone()).unwrap_or_else(|_| Self::put(items));
-	}
-
-	/// Read the length of the value in a fast way, without decoding the entire value.
-	///
-	/// `T` is required to implement `Codec::DecodeLength`.
-	///
-	/// Note that `0` is returned as the default value if no encoded value exists at the given key.
-	/// Therefore, this function cannot be used as a sign of _existence_. use the `::exists()`
-	/// function for this purpose.
 	fn decode_len() -> Result<usize, &'static str> where T: codec::DecodeLength, T: Len {
 		let key = Self::storage_value_final_key();
 

substrate/frame/support/src/storage/mod.rs

@@ -17,7 +17,7 @@
 //! Stuff to do with the runtime's storage.
 
 use sp_std::{prelude::*, marker::PhantomData};
-use codec::{FullCodec, FullEncode, Encode, EncodeAppend, EncodeLike, Decode};
+use codec::{FullCodec, FullEncode, Encode, EncodeLike, Decode};
 use crate::{traits::Len, hash::{Twox128, StorageHasher}};
 
 pub mod unhashed;
@@ -91,33 +91,18 @@ pub trait StorageValue<T: FullCodec> {
 
 	/// Append the given item to the value in the storage.
 	///
-	/// `T` is required to implement `codec::EncodeAppend`.
+	/// `T` is required to implement [`StorageAppend`].
-	fn append<Items, Item, EncodeLikeItem>(items: Items) -> Result<(), &'static str>
+	///
+	/// # Warning
+	///
+	/// If the storage item is not encoded properly, the storage item will be overwritten
+	/// and set to `[item]`. Any default value set for the storage item will be ignored
+	/// on overwrite.
+	fn append<Item, EncodeLikeItem>(item: EncodeLikeItem)
 	where
 		Item: Encode,
 		EncodeLikeItem: EncodeLike<Item>,
-		T: EncodeAppend<Item=Item>,
+		T: StorageAppend<Item>;
-		Items: IntoIterator<Item=EncodeLikeItem>,
-		Items::IntoIter: ExactSizeIterator;
-
-	/// Append the given items to the value in the storage.
-	///
-	/// `T` is required to implement `Codec::EncodeAppend`.
-	///
-	/// Upon any failure, it replaces `items` as the new value (assuming that the previous stored
-	/// data is simply corrupt and no longer usable).
-	///
-	/// ### WARNING
-	///
-	/// use with care; if your use-case is not _exactly_ as what this function is doing,
-	/// you should use append and sensibly handle failure within the runtime code if it happens.
-	fn append_or_put<Items, Item, EncodeLikeItem>(items: Items) where
-		Item: Encode,
-		EncodeLikeItem: EncodeLike<Item>,
-		T: EncodeAppend<Item=Item>,
-		Items: IntoIterator<Item=EncodeLikeItem> + Clone + EncodeLike<T>,
-		Items::IntoIter: ExactSizeIterator;
-
 
 	/// Read the length of the value in a fast way, without decoding the entire value.
 	///
@@ -176,25 +161,18 @@ pub trait StorageMap<K: FullEncode, V: FullCodec> {
 	/// Append the given items to the value in the storage.
 	///
 	/// `V` is required to implement `codec::EncodeAppend`.
-	fn append<Items, Item, EncodeLikeItem, KeyArg>(key: KeyArg, items: Items) -> Result<(), &'static str> where
-		KeyArg: EncodeLike<K>,
-		Item: Encode,
-		EncodeLikeItem: EncodeLike<Item>,
-		V: EncodeAppend<Item=Item>,
-		Items: IntoIterator<Item=EncodeLikeItem>,
-		Items::IntoIter: ExactSizeIterator;
-
-	/// Safely append the given items to the value in the storage. If a codec error occurs, then the
-	/// old (presumably corrupt) value is replaced with the given `items`.
 	///
-	/// `V` is required to implement `codec::EncodeAppend`.
+	/// # Warning
-	fn append_or_insert<Items, Item, EncodeLikeItem, KeyArg>(key: KeyArg, items: Items) where
+	///
-		KeyArg: EncodeLike<K>,
+	/// If the storage item is not encoded properly, the storage will be overwritten
+	/// and set to `[item]`. Any default value set for the storage item will be ignored
+	/// on overwrite.
+	fn append<Item, EncodeLikeItem, EncodeLikeKey>(key: EncodeLikeKey, item: EncodeLikeItem)
+	where
+		EncodeLikeKey: EncodeLike<K>,
 		Item: Encode,
 		EncodeLikeItem: EncodeLike<Item>,
-		V: EncodeAppend<Item=Item>,
+		V: StorageAppend<Item>;
-		Items: IntoIterator<Item=EncodeLikeItem> + Clone + EncodeLike<V>,
-		Items::IntoIter: ExactSizeIterator;
 
 	/// Read the length of the value in a fast way, without decoding the entire value.
 	///
@@ -351,38 +329,23 @@ pub trait StorageDoubleMap<K1: FullEncode, K2: FullEncode, V: FullCodec> {
 
 	/// Append the given item to the value in the storage.
 	///
-	/// `V` is required to implement `codec::EncodeAppend`.
+	/// `V` is required to implement [`StorageAppend`].
-	fn append<Items, Item, EncodeLikeItem, KArg1, KArg2>(
-		k1: KArg1,
-		k2: KArg2,
-		items: Items,
-	) -> Result<(), &'static str>
-	where
-		KArg1: EncodeLike<K1>,
-		KArg2: EncodeLike<K2>,
-		Item: Encode,
-		EncodeLikeItem: EncodeLike<Item>,
-		V: EncodeAppend<Item=Item>,
-		Items: IntoIterator<Item=EncodeLikeItem>,
-		Items::IntoIter: ExactSizeIterator;
-
-	/// Safely append the given items to the value in the storage. If a codec error occurs, then the
-	/// old (presumably corrupt) value is replaced with the given `items`.
 	///
-	/// `V` is required to implement `codec::EncodeAppend`.
+	/// # Warning
-	fn append_or_insert<Items, Item, EncodeLikeItem, KArg1, KArg2>(
+	///
+	/// If the storage item is not encoded properly, the storage will be overwritten
+	/// and set to `[item]`. Any default value set for the storage item will be ignored
+	/// on overwrite.
+	fn append<Item, EncodeLikeItem, KArg1, KArg2>(
 		k1: KArg1,
 		k2: KArg2,
-		items: Items,
+		item: EncodeLikeItem,
-	)
+	) where
-	where
 		KArg1: EncodeLike<K1>,
 		KArg2: EncodeLike<K2>,
 		Item: Encode,
 		EncodeLikeItem: EncodeLike<Item>,
-		V: EncodeAppend<Item=Item>,
+		V: StorageAppend<Item>;
-		Items: IntoIterator<Item=EncodeLikeItem> + Clone + EncodeLike<V>,
-		Items::IntoIter: ExactSizeIterator;
 
 	/// Read the length of the value in a fast way, without decoding the entire value.
 	///
@@ -449,7 +412,6 @@ impl<Value: Decode> Iterator for PrefixIterator<Value> {
 /// Twox128(module_prefix) ++ Twox128(storage_prefix)
 /// ```
 pub trait StoragePrefixedMap<Value: FullCodec> {
-
 	/// Module prefix. Used for generating final key.
 	fn module_prefix() -> &'static [u8];
 
@@ -525,6 +487,22 @@ pub trait StoragePrefixedMap<Value: FullCodec> {
 	}
 }
 
+/// Marker trait that will be implemented for types that support the `storage::append` api.
+///
+/// This trait is sealed.
+pub trait StorageAppend<Item: Encode>: private::Sealed {}
+
+/// Provides `Sealed` trait to prevent implementing trait `StorageAppend` outside of this crate.
+mod private {
+	use super::*;
+
+	pub trait Sealed {}
+
+	impl<T: Encode> Sealed for Vec<T> {}
+}
+
+impl<T: Encode> StorageAppend<T> for Vec<T> {}
+
 #[cfg(test)]
 mod test {
 	use sp_core::hashing::twox_128;

substrate/frame/support/test/tests/decl_storage.rs

@@ -526,50 +526,58 @@ mod test_append_and_len {
 	#[test]
 	fn append_works() {
 		TestExternalities::default().execute_with(|| {
-			let _ = MapVec::append(1, [1, 2, 3].iter());
+			for val in &[1, 2, 3, 4, 5] {
-			let _ = MapVec::append(1, [4, 5].iter());
+				MapVec::append(1, val);
+			}
 			assert_eq!(MapVec::get(1), vec![1, 2, 3, 4, 5]);
 
-			let _ = JustVec::append([1, 2, 3].iter());
+			MapVec::remove(1);
-			let _ = JustVec::append([4, 5].iter());
+			MapVec::append(1, 1);
+			assert_eq!(MapVec::get(1), vec![1]);
+
+			for val in &[1, 2, 3, 4, 5] {
+				JustVec::append(val);
+			}
 			assert_eq!(JustVec::get(), vec![1, 2, 3, 4, 5]);
+
+			JustVec::kill();
+			JustVec::append(1);
+			assert_eq!(JustVec::get(), vec![1]);
 		});
 	}
 
 	#[test]
-	fn append_works_for_default() {
+	fn append_overwrites_invalid_data() {
+		TestExternalities::default().execute_with(|| {
+			let key = JustVec::hashed_key();
+			// Set it to some invalid value.
+			frame_support::storage::unhashed::put_raw(&key, &*b"1");
+			assert_eq!(JustVec::get(), Vec::new());
+			assert_eq!(frame_support::storage::unhashed::get_raw(&key), Some(b"1".to_vec()));
+
+			JustVec::append(1);
+			JustVec::append(2);
+			assert_eq!(JustVec::get(), vec![1, 2]);
+		});
+	}
+
+	#[test]
+	fn append_overwrites_default() {
 		TestExternalities::default().execute_with(|| {
 			assert_eq!(JustVecWithDefault::get(), vec![6, 9]);
-			let _ = JustVecWithDefault::append([1].iter());
+			JustVecWithDefault::append(1);
-			assert_eq!(JustVecWithDefault::get(), vec![6, 9, 1]);
+			assert_eq!(JustVecWithDefault::get(), vec![1]);
 
 			assert_eq!(MapVecWithDefault::get(0), vec![6, 9]);
-			let _ = MapVecWithDefault::append(0, [1].iter());
+			MapVecWithDefault::append(0, 1);
-			assert_eq!(MapVecWithDefault::get(0), vec![6, 9, 1]);
+			assert_eq!(MapVecWithDefault::get(0), vec![1]);
 
 			assert_eq!(OptionVec::get(), None);
-			let _ = OptionVec::append([1].iter());
+			OptionVec::append(1);
 			assert_eq!(OptionVec::get(), Some(vec![1]));
 		});
 	}
 
-	#[test]
-	fn append_or_put_works() {
-		TestExternalities::default().execute_with(|| {
-			let _ = MapVec::append_or_insert(1, &[1, 2, 3][..]);
-			let _ = MapVec::append_or_insert(1, &[4, 5][..]);
-			assert_eq!(MapVec::get(1), vec![1, 2, 3, 4, 5]);
-
-			let _ = JustVec::append_or_put(&[1, 2, 3][..]);
-			let _ = JustVec::append_or_put(&[4, 5][..]);
-			assert_eq!(JustVec::get(), vec![1, 2, 3, 4, 5]);
-
-			let _ = OptionVec::append_or_put(&[1, 2, 3][..]);
-			let _ = OptionVec::append_or_put(&[4, 5][..]);
-			assert_eq!(OptionVec::get(), Some(vec![1, 2, 3, 4, 5]));
-		});
-	}
-
 	#[test]
 	fn len_works() {
 		TestExternalities::default().execute_with(|| {

substrate/frame/system/src/lib.rs

@@ -116,7 +116,7 @@ use sp_runtime::{
 use sp_core::{ChangesTrieConfiguration, storage::well_known_keys};
 use frame_support::{
 	decl_module, decl_event, decl_storage, decl_error, Parameter, ensure, debug,
-	storage::{self, generator::StorageValue},
+	storage,
 	traits::{
 		Contains, Get, ModuleToIndex, OnNewAccount, OnKilledAccount, IsDeadAccount, Happened,
 		StoredMap, EnsureOrigin,
@@ -852,16 +852,10 @@ impl<T: Trait> Module<T> {
 			old_event_count
 		};
 
-		// We use append api here to avoid bringing all events in the runtime when we push a
+		Events::<T>::append(&event);
- 		// new one in the list.
-		let encoded_event = event.encode();
-		sp_io::storage::append(&Events::<T>::storage_value_final_key()[..], encoded_event);
 
 		for topic in topics {
-			// The same applies here.
+			<EventTopics<T>>::append(topic, &(block_number, event_idx));
-			if <EventTopics<T>>::append(topic, &[(block_number, event_idx)]).is_err() {
-				return;
-			}
 		}
 	}
 

substrate/primitives/core/src/testing.rs

@@ -25,7 +25,6 @@ use crate::{
 };
 #[cfg(feature = "std")]
 use std::collections::HashSet;
-use codec::Encode;
 /// Key type for generic Ed25519 key.
 pub const ED25519: KeyTypeId = KeyTypeId(*b"ed25");
 /// Key type for generic Sr 25519 key.
@@ -172,6 +171,8 @@ impl crate::traits::BareCryptoStore for KeyStore {
 		key: &CryptoTypePublicPair,
 		msg: &[u8],
 	) -> Result<Vec<u8>, BareCryptoStoreError> {
+		use codec::Encode;
+
 		match key.0 {
 			ed25519::CRYPTO_ID => {
 				let key_pair: ed25519::Pair = self

substrate/primitives/io/src/lib.rs

@@ -117,7 +117,14 @@ pub trait Storage {
 		Externalities::clear_prefix(*self, prefix)
 	}
 
-	/// Append to storage item (assumes it is in "Vec" format).
+	/// Append the encoded `value` to the storage item at `key`.
+	///
+	/// The storage item needs to implement [`EncodeAppend`](codec::EncodeAppend).
+	///
+	/// # Warning
+	///
+	/// If the storage item does not support [`EncodeAppend`](codec::EncodeAppend) or
+	/// something else fails at appending, the storage item will be set to `[value]`.
 	fn append(&mut self, key: &[u8], value: Vec<u8>) {
 		self.storage_append(key.to_vec(), value);
 	}

substrate/primitives/state-machine/src/basic.rs

@@ -262,8 +262,8 @@ impl Externalities for BasicExternalities {
 		key: Vec<u8>,
 		value: Vec<u8>,
 	) {
-		let previous = self.inner.top.entry(key).or_default();
+		let current = self.inner.top.entry(key).or_default();
-		crate::ext::append_to_storage(previous, value).expect("Failed to append to storage");
+		crate::ext::StorageAppend::new(current).append(value);
 	}
 
 	fn chain_id(&self) -> u64 { 42 }

substrate/primitives/state-machine/src/ext.rs

@@ -30,7 +30,7 @@ use sp_core::{
 };
 use sp_trie::{trie_types::Layout, empty_child_trie_root};
 use sp_externalities::{Extensions, Extension};
-use codec::{Compact, Decode, Encode};
+use codec::{Decode, Encode, EncodeAppend};
 
 use std::{error, fmt, any::{Any, TypeId}};
 use log::{warn, trace};
@@ -428,7 +428,7 @@ where
 			&key,
 			|| backend.storage(&key).expect(EXT_NOT_ALLOWED_TO_FAIL).unwrap_or_default()
 		);
-		append_to_storage(current_value, value).expect(EXT_NOT_ALLOWED_TO_FAIL);
+		StorageAppend::new(current_value).append(value);
 	}
 
 	fn chain_id(&self) -> u64 {
@@ -549,16 +549,24 @@ where
 
 	fn wipe(&mut self) {
 		self.overlay.discard_prospective();
-		self.overlay.drain_storage_changes(&self.backend, None, Default::default(), self.storage_transaction_cache)
+		self.overlay.drain_storage_changes(
-			.expect(EXT_NOT_ALLOWED_TO_FAIL);
+			&self.backend,
+			None,
+			Default::default(),
+			self.storage_transaction_cache,
+		).expect(EXT_NOT_ALLOWED_TO_FAIL);
 		self.storage_transaction_cache.reset();
 		self.backend.wipe().expect(EXT_NOT_ALLOWED_TO_FAIL)
 	}
 
 	fn commit(&mut self) {
 		self.overlay.commit_prospective();
-		let changes = self.overlay.drain_storage_changes(&self.backend, None, Default::default(), self.storage_transaction_cache)
+		let changes = self.overlay.drain_storage_changes(
-			.expect(EXT_NOT_ALLOWED_TO_FAIL);
+			&self.backend,
+			None,
+			Default::default(),
+			self.storage_transaction_cache,
+		).expect(EXT_NOT_ALLOWED_TO_FAIL);
 		self.backend.commit(
 			changes.transaction_storage_root,
 			changes.transaction,
@@ -567,65 +575,43 @@ where
 	}
 }
 
-fn extract_length_data(data: &[u8]) -> Result<(u32, usize, usize), &'static str> {
-	use codec::CompactLen;
 
-	let len = u32::from(
+/// Implement `Encode` by forwarding the stored raw vec.
-		Compact::<u32>::decode(&mut &data[..])
+struct EncodeOpaqueValue(Vec<u8>);
-			.map_err(|_| "Incorrect updated item encoding")?
-	);
-	let new_len = len
-		.checked_add(1)
-		.ok_or_else(|| "New vec length greater than `u32::max_value()`.")?;
 
-	let encoded_len = Compact::<u32>::compact_len(&len);
+impl Encode for EncodeOpaqueValue {
-	let encoded_new_len = Compact::<u32>::compact_len(&new_len);
+	fn using_encoded<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
-
+		f(&self.0)
-	Ok((new_len, encoded_len, encoded_new_len))
+	}
 }
 
-pub fn append_to_storage(
+/// Auxialiary structure for appending a value to a storage item.
-	self_encoded: &mut Vec<u8>,
+pub(crate) struct StorageAppend<'a>(&'a mut Vec<u8>);
-	value: Vec<u8>,
+
-) -> Result<(), &'static str> {
+impl<'a> StorageAppend<'a> {
-	// No data present, just encode the given input data.
+	/// Create a new instance using the given `storage` reference.
-	if self_encoded.is_empty() {
+	pub fn new(storage: &'a mut Vec<u8>) -> Self {
-		Compact::from(1u32).encode_to(self_encoded);
+		Self(storage)
-		self_encoded.extend(value);
-		return Ok(());
 	}
 
-	let (new_len, encoded_len, encoded_new_len) = extract_length_data(&self_encoded)?;
+	/// Append the given `value` to the storage item.
+	///
+	/// If appending fails, `[value]` is stored in the storage item.
+	pub fn append(&mut self, value: Vec<u8>) {
+		let value = vec![EncodeOpaqueValue(value)];
 
-	let replace_len = |dest: &mut Vec<u8>| {
+		let item = std::mem::take(self.0);
-		Compact(new_len).using_encoded(|e| {
-			dest[..encoded_new_len].copy_from_slice(e);
-		})
-	};
 
-	let append_new_elems = |dest: &mut Vec<u8>| dest.extend(&value[..]);
+		*self.0 = match Vec::<EncodeOpaqueValue>::append_or_new(item, &value) {
-
+			Ok(item) => item,
-	// If old and new encoded len is equal, we don't need to copy the
+			Err(_) => {
-	// already encoded data.
+				log::error!(
-	if encoded_len == encoded_new_len {
+					target: "runtime",
-		replace_len(self_encoded);
+					"Failed to append value, resetting storage item to `[value]`.",
-		append_new_elems(self_encoded);
+				);
-
+				value.encode()
-		Ok(())
+			}
-	} else {
+		};
-		let size = encoded_new_len + self_encoded.len() - encoded_len;
-
-		let mut res = Vec::with_capacity(size + value.len());
-		unsafe { res.set_len(size); }
-
-		// Insert the new encoded len, copy the already encoded data and
-		// add the new element.
-		replace_len(&mut res);
-		res[encoded_new_len..size].copy_from_slice(&self_encoded[encoded_len..]);
-		append_new_elems(&mut res);
-		*self_encoded = res;
-
-		Ok(())
 	}
 }
 
@@ -902,4 +888,24 @@ mod tests {
 			Some(Blake2Hasher::hash(&[31]).as_ref().to_vec()),
 		);
 	}
+
+	#[test]
+	fn storage_append_works() {
+		let mut data = Vec::new();
+		let mut append = StorageAppend::new(&mut data);
+		append.append(1u32.encode());
+		append.append(2u32.encode());
+		drop(append);
+
+		assert_eq!(Vec::<u32>::decode(&mut &data[..]).unwrap(), vec![1, 2]);
+
+		// Initialize with some invalid data
+		let mut data = vec![1];
+		let mut append = StorageAppend::new(&mut data);
+		append.append(1u32.encode());
+		append.append(2u32.encode());
+		drop(append);
+
+		assert_eq!(Vec::<u32>::decode(&mut &data[..]).unwrap(), vec![1, 2]);
+	}
 }

substrate/primitives/storage/src/lib.rs

@@ -22,8 +22,7 @@
 use serde::{Serialize, Deserialize};
 use sp_debug_derive::RuntimeDebug;
 
-use sp_std::vec::Vec;
+use sp_std::{vec::Vec, ops::{Deref, DerefMut}};
-use sp_std::ops::{Deref, DerefMut};
 use ref_cast::RefCast;
 
 /// Storage key.
@@ -334,10 +333,15 @@ impl ChildTrieParentKeyId {
 	}
 }
 
-#[test]
+#[cfg(test)]
-fn test_prefix_default_child_info() {
+mod tests {
-	let child_info = ChildInfo::new_default(b"any key");
+	use super::*;
-	let prefix = child_info.child_type().parent_prefix();
+
-	assert!(prefix.starts_with(well_known_keys::CHILD_STORAGE_KEY_PREFIX));
+	#[test]
-	assert!(prefix.starts_with(well_known_keys::DEFAULT_CHILD_STORAGE_KEY_PREFIX));
+	fn test_prefix_default_child_info() {
+		let child_info = ChildInfo::new_default(b"any key");
+		let prefix = child_info.child_type().parent_prefix();
+		assert!(prefix.starts_with(well_known_keys::CHILD_STORAGE_KEY_PREFIX));
+		assert!(prefix.starts_with(well_known_keys::DEFAULT_CHILD_STORAGE_KEY_PREFIX));
+	}
 }

substrate/primitives/wasm-interface/src/lib.rs

@@ -119,7 +119,7 @@ mod private {
 /// Something that can be wrapped in a wasm `Pointer`.
 ///
 /// This trait is sealed.
-pub trait PointerType: Sized {
+pub trait PointerType: Sized + private::Sealed {
 	/// The size of the type in wasm.
 	const SIZE: u32 = mem::size_of::<Self>() as u32;
 }