diff --git a/substrate/client/db/src/bench.rs b/substrate/client/db/src/bench.rs index f3c8f1aff9..5696922b4f 100644 --- a/substrate/client/db/src/bench.rs +++ b/substrate/client/db/src/bench.rs @@ -350,13 +350,13 @@ impl StateBackend> for BenchmarkingState { } } - fn for_keys_in_child_storage( + fn apply_to_child_keys_while bool>( &self, child_info: &ChildInfo, f: F, ) { if let Some(ref state) = *self.state.borrow() { - state.for_keys_in_child_storage(child_info, f) + state.apply_to_child_keys_while(child_info, f) } } diff --git a/substrate/client/db/src/lib.rs b/substrate/client/db/src/lib.rs index 8254e652f6..e32e45a2f3 100644 --- a/substrate/client/db/src/lib.rs +++ b/substrate/client/db/src/lib.rs @@ -195,12 +195,12 @@ impl StateBackend> for RefTrackingState { self.state.for_key_values_with_prefix(prefix, f) } - fn for_keys_in_child_storage( + fn apply_to_child_keys_while bool>( &self, child_info: &ChildInfo, f: F, ) { - self.state.for_keys_in_child_storage(child_info, f) + self.state.apply_to_child_keys_while(child_info, f) } fn for_child_keys_with_prefix( diff --git a/substrate/client/db/src/storage_cache.rs b/substrate/client/db/src/storage_cache.rs index 0b4b6d4f88..292d3c5162 100644 --- a/substrate/client/db/src/storage_cache.rs +++ b/substrate/client/db/src/storage_cache.rs @@ -584,12 +584,12 @@ impl>, B: BlockT> StateBackend> for Cachin self.state.exists_child_storage(child_info, key) } - fn for_keys_in_child_storage( + fn apply_to_child_keys_while bool>( &self, child_info: &ChildInfo, f: F, ) { - self.state.for_keys_in_child_storage(child_info, f) + self.state.apply_to_child_keys_while(child_info, f) } fn next_storage_key(&self, key: &[u8]) -> Result>, Self::Error> { @@ -766,12 +766,12 @@ impl>, B: BlockT> StateBackend> for Syncin self.caching_state().exists_child_storage(child_info, key) } - fn for_keys_in_child_storage( + fn apply_to_child_keys_while bool>( &self, child_info: &ChildInfo, f: F, ) { - self.caching_state().for_keys_in_child_storage(child_info, f) + self.caching_state().apply_to_child_keys_while(child_info, f) } fn next_storage_key(&self, key: &[u8]) -> Result>, Self::Error> { diff --git a/substrate/client/light/src/backend.rs b/substrate/client/light/src/backend.rs index be7953e528..74e1d613bc 100644 --- a/substrate/client/light/src/backend.rs +++ b/substrate/client/light/src/backend.rs @@ -441,14 +441,14 @@ impl StateBackend for GenesisOrUnavailableState } } - fn for_keys_in_child_storage( + fn apply_to_child_keys_while bool>( &self, child_info: &ChildInfo, action: A, ) { match *self { GenesisOrUnavailableState::Genesis(ref state) => - state.for_keys_in_child_storage(child_info, action), + state.apply_to_child_keys_while(child_info, action), GenesisOrUnavailableState::Unavailable => (), } } diff --git a/substrate/frame/contracts/src/rent.rs b/substrate/frame/contracts/src/rent.rs index 6ee65a54bb..8b6f81c916 100644 --- a/substrate/frame/contracts/src/rent.rs +++ b/substrate/frame/contracts/src/rent.rs @@ -240,6 +240,7 @@ where >::remove(account); child::kill_storage( &alive_contract_info.child_trie_info(), + None, ); >::deposit_event(RawEvent::Evicted(account.clone(), false)); None @@ -263,6 +264,7 @@ where child::kill_storage( &alive_contract_info.child_trie_info(), + None, ); >::deposit_event(RawEvent::Evicted(account.clone(), true)); diff --git a/substrate/frame/contracts/src/storage.rs b/substrate/frame/contracts/src/storage.rs index c9eeba4633..ba09adb285 100644 --- a/substrate/frame/contracts/src/storage.rs +++ b/substrate/frame/contracts/src/storage.rs @@ -195,7 +195,7 @@ where /// This function doesn't affect the account. pub fn destroy_contract(address: &AccountIdOf, trie_id: &TrieId) { >::remove(address); - child::kill_storage(&crate::child_trie_info(&trie_id)); + child::kill_storage(&crate::child_trie_info(&trie_id), None); } /// This generator uses inner counter for account id and applies the hash over `AccountId + diff --git a/substrate/frame/support/src/storage/child.rs b/substrate/frame/support/src/storage/child.rs index 431b5e0930..d986155447 100644 --- a/substrate/frame/support/src/storage/child.rs +++ b/substrate/frame/support/src/storage/child.rs @@ -25,6 +25,14 @@ use crate::sp_std::prelude::*; use codec::{Codec, Encode, Decode}; pub use sp_core::storage::{ChildInfo, ChildType}; +/// The outcome of calling [`kill_storage`]. +pub enum KillOutcome { + /// No key remains in the child trie. + AllRemoved, + /// At least one key still resides in the child trie due to the supplied limit. + SomeRemaining, +} + /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry. pub fn get( child_info: &ChildInfo, @@ -148,13 +156,37 @@ pub fn exists( } /// Remove all `storage_key` key/values +/// +/// Deletes all keys from the overlay and up to `limit` keys from the backend if +/// it is set to `Some`. No limit is applied when `limit` is set to `None`. +/// +/// The limit can be used to partially delete a child trie in case it is too large +/// to delete in one go (block). +/// +/// # Note +/// +/// Please note that keys that are residing in the overlay for that child trie when +/// issuing this call are all deleted without counting towards the `limit`. Only keys +/// written during the current block are part of the overlay. Deleting with a `limit` +/// mostly makes sense with an empty overlay for that child trie. +/// +/// Calling this function multiple times per block for the same `storage_key` does +/// not make much sense because it is not cumulative when called inside the same block. +/// Use this function to distribute the deletion of a single child trie across multiple +/// blocks. pub fn kill_storage( child_info: &ChildInfo, -) { - match child_info.child_type() { + limit: Option, +) -> KillOutcome { + let all_removed = match child_info.child_type() { ChildType::ParentKeyId => sp_io::default_child_storage::storage_kill( child_info.storage_key(), + limit ), + }; + match all_removed { + true => KillOutcome::AllRemoved, + false => KillOutcome::SomeRemaining, } } diff --git a/substrate/primitives/externalities/src/lib.rs b/substrate/primitives/externalities/src/lib.rs index 388482964f..6869969f4b 100644 --- a/substrate/primitives/externalities/src/lib.rs +++ b/substrate/primitives/externalities/src/lib.rs @@ -137,7 +137,17 @@ pub trait Externalities: ExtensionStore { ) -> Option>; /// Clear an entire child storage. - fn kill_child_storage(&mut self, child_info: &ChildInfo); + /// + /// Deletes all keys from the overlay and up to `limit` keys from the backend. No + /// limit is applied if `limit` is `None`. Returns `true` if the child trie was + /// removed completely and `false` if there are remaining keys after the function + /// returns. + /// + /// # Note + /// + /// An implementation is free to delete more keys than the specified limit as long as + /// it is able to do that in constant time. + fn kill_child_storage(&mut self, child_info: &ChildInfo, limit: Option) -> bool; /// Clear storage entries which keys are start with the given prefix. fn clear_prefix(&mut self, prefix: &[u8]); diff --git a/substrate/primitives/io/src/lib.rs b/substrate/primitives/io/src/lib.rs index d812baefb5..b6ae64e5f8 100644 --- a/substrate/primitives/io/src/lib.rs +++ b/substrate/primitives/io/src/lib.rs @@ -279,7 +279,35 @@ pub trait DefaultChildStorage { storage_key: &[u8], ) { let child_info = ChildInfo::new_default(storage_key); - self.kill_child_storage(&child_info); + self.kill_child_storage(&child_info, None); + } + + /// Clear a child storage key. + /// + /// Deletes all keys from the overlay and up to `limit` keys from the backend if + /// it is set to `Some`. No limit is applied when `limit` is set to `None`. + /// + /// The limit can be used to partially delete a child trie in case it is too large + /// to delete in one go (block). + /// + /// It returns false iff some keys are remaining in + /// the child trie after the functions returns. + /// + /// # Note + /// + /// Please note that keys that are residing in the overlay for that child trie when + /// issuing this call are all deleted without counting towards the `limit`. Only keys + /// written during the current block are part of the overlay. Deleting with a `limit` + /// mostly makes sense with an empty overlay for that child trie. + /// + /// Calling this function multiple times per block for the same `storage_key` does + /// not make much sense because it is not cumulative when called inside the same block. + /// Use this function to distribute the deletion of a single child trie across multiple + /// blocks. + #[version(2)] + fn storage_kill(&mut self, storage_key: &[u8], limit: Option) -> bool { + let child_info = ChildInfo::new_default(storage_key); + self.kill_child_storage(&child_info, limit) } /// Check a child storage key. diff --git a/substrate/primitives/state-machine/src/backend.rs b/substrate/primitives/state-machine/src/backend.rs index 360fe9a985..02151c2480 100644 --- a/substrate/primitives/state-machine/src/backend.rs +++ b/substrate/primitives/state-machine/src/backend.rs @@ -94,7 +94,8 @@ pub trait Backend: sp_std::fmt::Debug { ) -> Result, Self::Error>; /// Retrieve all entries keys of child storage and call `f` for each of those keys. - fn for_keys_in_child_storage( + /// Aborts as soon as `f` returns false. + fn apply_to_child_keys_while bool>( &self, child_info: &ChildInfo, f: F, @@ -263,12 +264,12 @@ impl<'a, T: Backend, H: Hasher> Backend for &'a T { (*self).child_storage(child_info, key) } - fn for_keys_in_child_storage( + fn apply_to_child_keys_while bool>( &self, child_info: &ChildInfo, f: F, ) { - (*self).for_keys_in_child_storage(child_info, f) + (*self).apply_to_child_keys_while(child_info, f) } fn next_storage_key(&self, key: &[u8]) -> Result, Self::Error> { diff --git a/substrate/primitives/state-machine/src/basic.rs b/substrate/primitives/state-machine/src/basic.rs index 5e3c9bed64..9de75785e4 100644 --- a/substrate/primitives/state-machine/src/basic.rs +++ b/substrate/primitives/state-machine/src/basic.rs @@ -210,8 +210,10 @@ impl Externalities for BasicExternalities { fn kill_child_storage( &mut self, child_info: &ChildInfo, - ) { + _limit: Option, + ) -> bool { self.inner.children_default.remove(child_info.storage_key()); + true } fn clear_prefix(&mut self, prefix: &[u8]) { @@ -407,7 +409,7 @@ mod tests { ext.clear_child_storage(child_info, b"dog"); assert_eq!(ext.child_storage(child_info, b"dog"), None); - ext.kill_child_storage(child_info); + ext.kill_child_storage(child_info, None); assert_eq!(ext.child_storage(child_info, b"doe"), None); } diff --git a/substrate/primitives/state-machine/src/ext.rs b/substrate/primitives/state-machine/src/ext.rs index 53aab42999..3c4d88f392 100644 --- a/substrate/primitives/state-machine/src/ext.rs +++ b/substrate/primitives/state-machine/src/ext.rs @@ -411,18 +411,41 @@ where fn kill_child_storage( &mut self, child_info: &ChildInfo, - ) { + limit: Option, + ) -> bool { trace!(target: "state", "{:04x}: KillChild({})", self.id, HexDisplay::from(&child_info.storage_key()), ); let _guard = guard(); - self.mark_dirty(); self.overlay.clear_child_storage(child_info); - self.backend.for_keys_in_child_storage(child_info, |key| { - self.overlay.set_child_storage(child_info, key.to_vec(), None); - }); + + if let Some(limit) = limit { + let mut num_deleted: u32 = 0; + let mut all_deleted = true; + self.backend.apply_to_child_keys_while(child_info, |key| { + if num_deleted == limit { + all_deleted = false; + return false; + } + if let Some(num) = num_deleted.checked_add(1) { + num_deleted = num; + } else { + all_deleted = false; + return false; + } + self.overlay.set_child_storage(child_info, key.to_vec(), None); + true + }); + all_deleted + } else { + self.backend.apply_to_child_keys_while(child_info, |key| { + self.overlay.set_child_storage(child_info, key.to_vec(), None); + true + }); + true + } } fn clear_prefix(&mut self, prefix: &[u8]) { diff --git a/substrate/primitives/state-machine/src/lib.rs b/substrate/primitives/state-machine/src/lib.rs index 28148b6411..c83dce4bed 100644 --- a/substrate/primitives/state-machine/src/lib.rs +++ b/substrate/primitives/state-machine/src/lib.rs @@ -1147,6 +1147,86 @@ mod tests { ); } + #[test] + fn limited_child_kill_works() { + let child_info = ChildInfo::new_default(b"sub1"); + let initial: HashMap<_, BTreeMap<_, _>> = map![ + Some(child_info.clone()) => map![ + b"a".to_vec() => b"0".to_vec(), + b"b".to_vec() => b"1".to_vec(), + b"c".to_vec() => b"2".to_vec(), + b"d".to_vec() => b"3".to_vec() + ], + ]; + let backend = InMemoryBackend::::from(initial); + + let mut overlay = OverlayedChanges::default(); + overlay.set_child_storage(&child_info, b"1".to_vec(), Some(b"1312".to_vec())); + overlay.set_child_storage(&child_info, b"2".to_vec(), Some(b"1312".to_vec())); + overlay.set_child_storage(&child_info, b"3".to_vec(), Some(b"1312".to_vec())); + overlay.set_child_storage(&child_info, b"4".to_vec(), Some(b"1312".to_vec())); + + { + let mut offchain_overlay = Default::default(); + let mut cache = StorageTransactionCache::default(); + let mut ext = Ext::new( + &mut overlay, + &mut offchain_overlay, + &mut cache, + &backend, + changes_trie::disabled_state::<_, u64>(), + None, + ); + assert_eq!(ext.kill_child_storage(&child_info, Some(2)), false); + } + + assert_eq!( + overlay.children() + .flat_map(|(iter, _child_info)| iter) + .map(|(k, v)| (k.clone(), v.value().clone())) + .collect::>(), + map![ + b"1".to_vec() => None.into(), + b"2".to_vec() => None.into(), + b"3".to_vec() => None.into(), + b"4".to_vec() => None.into(), + b"a".to_vec() => None.into(), + b"b".to_vec() => None.into(), + ], + ); + } + + #[test] + fn limited_child_kill_off_by_one_works() { + let child_info = ChildInfo::new_default(b"sub1"); + let initial: HashMap<_, BTreeMap<_, _>> = map![ + Some(child_info.clone()) => map![ + b"a".to_vec() => b"0".to_vec(), + b"b".to_vec() => b"1".to_vec(), + b"c".to_vec() => b"2".to_vec(), + b"d".to_vec() => b"3".to_vec() + ], + ]; + let backend = InMemoryBackend::::from(initial); + let mut overlay = OverlayedChanges::default(); + let mut offchain_overlay = Default::default(); + let mut cache = StorageTransactionCache::default(); + let mut ext = Ext::new( + &mut overlay, + &mut offchain_overlay, + &mut cache, + &backend, + changes_trie::disabled_state::<_, u64>(), + None, + ); + assert_eq!(ext.kill_child_storage(&child_info, Some(0)), false); + assert_eq!(ext.kill_child_storage(&child_info, Some(1)), false); + assert_eq!(ext.kill_child_storage(&child_info, Some(2)), false); + assert_eq!(ext.kill_child_storage(&child_info, Some(3)), false); + assert_eq!(ext.kill_child_storage(&child_info, Some(4)), true); + assert_eq!(ext.kill_child_storage(&child_info, Some(5)), true); + } + #[test] fn set_child_storage_works() { let child_info = ChildInfo::new_default(b"sub1"); @@ -1179,6 +1259,7 @@ mod tests { ); ext.kill_child_storage( child_info, + None, ); assert_eq!( ext.child_storage( diff --git a/substrate/primitives/state-machine/src/proving_backend.rs b/substrate/primitives/state-machine/src/proving_backend.rs index 0888c561ca..63a027cfba 100644 --- a/substrate/primitives/state-machine/src/proving_backend.rs +++ b/substrate/primitives/state-machine/src/proving_backend.rs @@ -204,12 +204,12 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> self.0.child_storage(child_info, key) } - fn for_keys_in_child_storage( + fn apply_to_child_keys_while bool>( &self, child_info: &ChildInfo, f: F, ) { - self.0.for_keys_in_child_storage(child_info, f) + self.0.apply_to_child_keys_while(child_info, f) } fn next_storage_key(&self, key: &[u8]) -> Result>, Self::Error> { diff --git a/substrate/primitives/state-machine/src/read_only.rs b/substrate/primitives/state-machine/src/read_only.rs index 1b70958145..2ab92f5fbb 100644 --- a/substrate/primitives/state-machine/src/read_only.rs +++ b/substrate/primitives/state-machine/src/read_only.rs @@ -131,7 +131,8 @@ impl<'a, H: Hasher, B: 'a + Backend> Externalities for ReadOnlyExternalities< fn kill_child_storage( &mut self, _child_info: &ChildInfo, - ) { + _limit: Option, + ) -> bool { unimplemented!("kill_child_storage is not supported in ReadOnlyExternalities") } diff --git a/substrate/primitives/state-machine/src/trie_backend.rs b/substrate/primitives/state-machine/src/trie_backend.rs index 4eaa0870ba..ffae1a02c0 100644 --- a/substrate/primitives/state-machine/src/trie_backend.rs +++ b/substrate/primitives/state-machine/src/trie_backend.rs @@ -113,12 +113,12 @@ impl, H: Hasher> Backend for TrieBackend where self.essence.for_key_values_with_prefix(prefix, f) } - fn for_keys_in_child_storage( + fn apply_to_child_keys_while bool>( &self, child_info: &ChildInfo, f: F, ) { - self.essence.for_keys_in_child_storage(child_info, f) + self.essence.apply_to_child_keys_while(child_info, f) } fn for_child_keys_with_prefix( diff --git a/substrate/primitives/state-machine/src/trie_backend_essence.rs b/substrate/primitives/state-machine/src/trie_backend_essence.rs index 37bbbb7cf9..8485cb27e7 100644 --- a/substrate/primitives/state-machine/src/trie_backend_essence.rs +++ b/substrate/primitives/state-machine/src/trie_backend_essence.rs @@ -190,7 +190,8 @@ impl, H: Hasher> TrieBackendEssence where H::Out: } /// Retrieve all entries keys of child storage and call `f` for each of those keys. - pub fn for_keys_in_child_storage( + /// Aborts as soon as `f` returns false. + pub fn apply_to_child_keys_while bool>( &self, child_info: &ChildInfo, f: F, diff --git a/substrate/primitives/tasks/src/async_externalities.rs b/substrate/primitives/tasks/src/async_externalities.rs index 8994d069e4..efb4c498f7 100644 --- a/substrate/primitives/tasks/src/async_externalities.rs +++ b/substrate/primitives/tasks/src/async_externalities.rs @@ -118,7 +118,8 @@ impl Externalities for AsyncExternalities { fn kill_child_storage( &mut self, _child_info: &ChildInfo, - ) { + _limit: Option, + ) -> bool { panic!("`kill_child_storage`: should not be used in async externalities!") } diff --git a/substrate/primitives/trie/src/lib.rs b/substrate/primitives/trie/src/lib.rs index 73a4a8029b..2687d8e422 100644 --- a/substrate/primitives/trie/src/lib.rs +++ b/substrate/primitives/trie/src/lib.rs @@ -271,7 +271,8 @@ pub fn child_delta_trie_root( } /// Call `f` for all keys in a child trie. -pub fn for_keys_in_child_trie( +/// Aborts as soon as `f` returns false. +pub fn for_keys_in_child_trie bool, DB>( keyspace: &[u8], db: &DB, root_slice: &[u8], @@ -290,7 +291,9 @@ pub fn for_keys_in_child_trie( for x in iter { let (key, _) = x?; - f(&key); + if !f(&key) { + break; + } } Ok(())