Safe and sane multi-item storage removal (#11490)

* Fix overlay prefix removal result

* Second part of the overlay prefix removal fix.

* Report only items deleted from storage in clear_prefix

* Fix kill_prefix

* Formatting

* Remove unused code

* Fixes

* Fixes

* Introduce clear_prefix host function v3

* Formatting

* Use v2 for now

* Fixes

* Formatting

* Docs

* Child prefix removal should also hide v3 for now

* Fixes

* Fixes

* Formatting

* Fixes

* apply_to_keys_whle takes start_at

* apply_to_keys_whle takes start_at

* apply_to_keys_whle takes start_at

* Cursor API; force limits

* Use unsafe deprecated functions

* Formatting

* Fixes

* Grumbles

* Fixes

* Docs

* Some nitpicks 🙈

* Update primitives/externalities/src/lib.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Formatting

* Fixes

* cargo fmt

* Fixes

* Update primitives/io/src/lib.rs

Co-authored-by: Keith Yeung <kungfukeith11@gmail.com>

* Formatting

* Fixes

Co-authored-by: Bastian Köcher <info@kchr.de>
Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
Co-authored-by: Keith Yeung <kungfukeith11@gmail.com>

substrate/primitives/externalities/src/lib.rs

@@ -52,6 +52,30 @@ pub enum Error {
 	StorageUpdateFailed(&'static str),
 }
 
+/// Results concerning an operation to remove many keys.
+#[derive(codec::Encode, codec::Decode)]
+#[must_use]
+pub struct MultiRemovalResults {
+	/// A continuation cursor which, if `Some` must be provided to the subsequent removal call.
+	/// If `None` then all removals are complete and no further calls are needed.
+	pub maybe_cursor: Option<Vec<u8>>,
+	/// The number of items removed from the backend database.
+	pub backend: u32,
+	/// The number of unique keys removed, taking into account both the backend and the overlay.
+	pub unique: u32,
+	/// The number of iterations (each requiring a storage seek/read) which were done.
+	pub loops: u32,
+}
+
+impl MultiRemovalResults {
+	/// Deconstruct into the internal components.
+	///
+	/// Returns `(maybe_cursor, backend, unique, loops)`.
+	pub fn deconstruct(self) -> (Option<Vec<u8>>, u32, u32, u32) {
+		(self.maybe_cursor, self.backend, self.unique, self.loops)
+	}
+}
+
 /// The Substrate externalities.
 ///
 /// Provides access to the storage and to other registered extensions.
@@ -118,32 +142,47 @@ pub trait Externalities: ExtensionStore {
 
 	/// Clear an entire child storage.
 	///
-	/// Deletes all keys from the overlay and up to `limit` keys from the backend. No
-	/// limit is applied if `limit` is `None`. Returned boolean is `true` if the child trie was
-	/// removed completely and `false` if there are remaining keys after the function
-	/// returns. Returned `u32` is the number of keys that was removed at the end of the
-	/// operation.
+	/// Deletes all keys from the overlay and up to `maybe_limit` keys from the backend. No
+	/// limit is applied if `maybe_limit` is `None`. Returns the cursor for the next call as `Some`
+	/// if the child trie deletion operation is incomplete. In this case, it should be passed into
+	/// the next call to avoid unaccounted iterations on the backend. Returns also the the number
+	/// of keys that were removed from the backend, the number of unique keys removed in total
+	/// (including from the overlay) and the number of backend iterations done.
+	///
+	/// As long as `maybe_cursor` is passed from the result of the previous call, then the number of
+	/// iterations done will only ever be one more than the number of keys removed.
 	///
 	/// # Note
 	///
 	/// An implementation is free to delete more keys than the specified limit as long as
 	/// it is able to do that in constant time.
-	fn kill_child_storage(&mut self, child_info: &ChildInfo, limit: Option<u32>) -> (bool, u32);
+	fn kill_child_storage(
+		&mut self,
+		child_info: &ChildInfo,
+		maybe_limit: Option<u32>,
+		maybe_cursor: Option<&[u8]>,
+	) -> MultiRemovalResults;
 
 	/// Clear storage entries which keys are start with the given prefix.
 	///
-	/// `limit` and result works as for `kill_child_storage`.
-	fn clear_prefix(&mut self, prefix: &[u8], limit: Option<u32>) -> (bool, u32);
+	/// `maybe_limit`, `maybe_cursor` and result works as for `kill_child_storage`.
+	fn clear_prefix(
+		&mut self,
+		prefix: &[u8],
+		maybe_limit: Option<u32>,
+		maybe_cursor: Option<&[u8]>,
+	) -> MultiRemovalResults;
 
 	/// Clear child storage entries which keys are start with the given prefix.
 	///
-	/// `limit` and result works as for `kill_child_storage`.
+	/// `maybe_limit`, `maybe_cursor` and result works as for `kill_child_storage`.
 	fn clear_child_prefix(
 		&mut self,
 		child_info: &ChildInfo,
 		prefix: &[u8],
-		limit: Option<u32>,
-	) -> (bool, u32);
+		maybe_limit: Option<u32>,
+		maybe_cursor: Option<&[u8]>,
+	) -> MultiRemovalResults;
 
 	/// Set or clear a storage entry (`key`) of current contract being called (effective
 	/// immediately).