PoV Reclaim (Clawback) Node Side (#1462)

This PR provides the infrastructure for the pov-reclaim mechanism discussed in #209. The goal is to provide the current proof size to the runtime so it can be used to reclaim storage weight. ## New Host Function - A new host function is provided [here](https://github.com/skunert/polkadot-sdk/blob/5b317fda3be205f4136f10d4490387ccd4f9765d/cumulus/primitives/pov-reclaim/src/lib.rs#L23). It returns the size of the current proof size to the runtime. If recording is not enabled, it returns 0. ## Implementation Overview - Implement option to enable proof recording during import in the client. This is currently enabled for `polkadot-parachain`, `parachain-template` and the cumulus test node. - Make the proof recorder ready for no-std. It was previously only enabled for std environments, but we need to record the proof size in `validate_block` too. - Provide a recorder implementation that only the records the size of incoming nodes and does not store the nodes itself. - Fix benchmarks that were broken by async backing changes - Provide new externalities extension that is registered by default if proof recording is enabled. - I think we should discuss the naming, pov-reclaim was more intuitive to me, but we could also go with clawback like in the issue. ## Impact of proof recording during import With proof recording: 6.3058 Kelem/s Without proof recording: 6.3427 Kelem/s The measured impact on the importing performance is quite low on my machine using the block import benchmark. With proof recording I am seeing a performance hit of 0.585%. --------- Co-authored-by: command-bot <> Co-authored-by: Davide Galassi <davxy@datawok.net> Co-authored-by: Bastian Köcher <git@kchr.de>
2026-05-30 12:51:02 +00:00 · 2023-11-30 15:56:34 +01:00
parent 64361ac19a
commit 9a650c46fd
34 changed files with 895 additions and 237 deletions
@@ -33,12 +33,12 @@ use sp_core::storage::{ChildInfo, StateVersion};
 #[cfg(feature = "std")]
 use sp_trie::{
 	cache::{LocalTrieCache, TrieCache},
-	recorder::Recorder,
-	MemoryDB, StorageProof,
+	MemoryDB,
 };
 #[cfg(not(feature = "std"))]
 use sp_trie::{Error, NodeCodec};
-use sp_trie::{MerkleValue, PrefixedMemoryDB};
+use sp_trie::{MerkleValue, PrefixedMemoryDB, StorageProof, TrieRecorderProvider};
+
 use trie_db::TrieCache as TrieCacheT;
 #[cfg(not(feature = "std"))]
 use trie_db::{node::NodeOwned, CachedValue};
@@ -112,8 +112,6 @@ pub struct UnimplementedCacheProvider<H> {
 	// Not strictly necessary, but the H bound allows to use this as a drop-in
 	// replacement for the `LocalTrieCache` in no-std contexts.
 	_phantom: core::marker::PhantomData<H>,
-	// Statically prevents construction.
-	_infallible: core::convert::Infallible,
 }

 #[cfg(not(feature = "std"))]
@@ -156,52 +154,83 @@ impl<H: Hasher> TrieCacheProvider<H> for UnimplementedCacheProvider<H> {
 	}
 }

+/// Recorder provider that allows construction of a [`TrieBackend`] and satisfies the requirements,
+/// but can never be instantiated.
+#[cfg(not(feature = "std"))]
+pub struct UnimplementedRecorderProvider<H> {
+	// Not strictly necessary, but the H bound allows to use this as a drop-in
+	// replacement for the [`sp_trie::recorder::Recorder`] in no-std contexts.
+	_phantom: core::marker::PhantomData<H>,
+}
+
+#[cfg(not(feature = "std"))]
+impl<H: Hasher> trie_db::TrieRecorder<H::Out> for UnimplementedRecorderProvider<H> {
+	fn record<'a>(&mut self, _access: trie_db::TrieAccess<'a, H::Out>) {
+		unimplemented!()
+	}
+
+	fn trie_nodes_recorded_for_key(&self, _key: &[u8]) -> trie_db::RecordedForKey {
+		unimplemented!()
+	}
+}
+
+#[cfg(not(feature = "std"))]
+impl<H: Hasher> TrieRecorderProvider<H> for UnimplementedRecorderProvider<H> {
+	type Recorder<'a> = UnimplementedRecorderProvider<H> where H: 'a;
+
+	fn drain_storage_proof(self) -> Option<StorageProof> {
+		unimplemented!()
+	}
+
+	fn as_trie_recorder(&self, _storage_root: H::Out) -> Self::Recorder<'_> {
+		unimplemented!()
+	}
+}
+
 #[cfg(feature = "std")]
 type DefaultCache<H> = LocalTrieCache<H>;

 #[cfg(not(feature = "std"))]
 type DefaultCache<H> = UnimplementedCacheProvider<H>;

+#[cfg(feature = "std")]
+type DefaultRecorder<H> = sp_trie::recorder::Recorder<H>;
+
+#[cfg(not(feature = "std"))]
+type DefaultRecorder<H> = UnimplementedRecorderProvider<H>;
+
 /// Builder for creating a [`TrieBackend`].
-pub struct TrieBackendBuilder<S: TrieBackendStorage<H>, H: Hasher, C = DefaultCache<H>> {
+pub struct TrieBackendBuilder<
+	S: TrieBackendStorage<H>,
+	H: Hasher,
+	C = DefaultCache<H>,
+	R = DefaultRecorder<H>,
+> {
 	storage: S,
 	root: H::Out,
-	#[cfg(feature = "std")]
-	recorder: Option<Recorder<H>>,
+	recorder: Option<R>,
 	cache: Option<C>,
 }

-impl<S, H> TrieBackendBuilder<S, H, DefaultCache<H>>
+impl<S, H> TrieBackendBuilder<S, H>
 where
 	S: TrieBackendStorage<H>,
 	H: Hasher,
 {
 	/// Create a new builder instance.
 	pub fn new(storage: S, root: H::Out) -> Self {
-		Self {
-			storage,
-			root,
-			#[cfg(feature = "std")]
-			recorder: None,
-			cache: None,
-		}
+		Self { storage, root, recorder: None, cache: None }
 	}
 }

-impl<S, H, C> TrieBackendBuilder<S, H, C>
+impl<S, H, C, R> TrieBackendBuilder<S, H, C, R>
 where
 	S: TrieBackendStorage<H>,
 	H: Hasher,
 {
 	/// Create a new builder instance.
 	pub fn new_with_cache(storage: S, root: H::Out, cache: C) -> Self {
-		Self {
-			storage,
-			root,
-			#[cfg(feature = "std")]
-			recorder: None,
-			cache: Some(cache),
-		}
+		Self { storage, root, recorder: None, cache: Some(cache) }
 	}
 	/// Wrap the given [`TrieBackend`].
 	///
@@ -210,53 +239,47 @@ where
 	/// backend.
 	///
 	/// The backend storage and the cache will be taken from `other`.
-	pub fn wrap(other: &TrieBackend<S, H, C>) -> TrieBackendBuilder<&S, H, &C> {
+	pub fn wrap(other: &TrieBackend<S, H, C, R>) -> TrieBackendBuilder<&S, H, &C, R> {
 		TrieBackendBuilder {
 			storage: other.essence.backend_storage(),
 			root: *other.essence.root(),
-			#[cfg(feature = "std")]
 			recorder: None,
 			cache: other.essence.trie_node_cache.as_ref(),
 		}
 	}

 	/// Use the given optional `recorder` for the to be configured [`TrieBackend`].
-	#[cfg(feature = "std")]
-	pub fn with_optional_recorder(self, recorder: Option<Recorder<H>>) -> Self {
+	pub fn with_optional_recorder(self, recorder: Option<R>) -> Self {
 		Self { recorder, ..self }
 	}

 	/// Use the given `recorder` for the to be configured [`TrieBackend`].
-	#[cfg(feature = "std")]
-	pub fn with_recorder(self, recorder: Recorder<H>) -> Self {
+	pub fn with_recorder(self, recorder: R) -> Self {
 		Self { recorder: Some(recorder), ..self }
 	}

 	/// Use the given optional `cache` for the to be configured [`TrieBackend`].
-	pub fn with_optional_cache<LC>(self, cache: Option<LC>) -> TrieBackendBuilder<S, H, LC> {
+	pub fn with_optional_cache<LC>(self, cache: Option<LC>) -> TrieBackendBuilder<S, H, LC, R> {
 		TrieBackendBuilder {
 			cache,
 			root: self.root,
 			storage: self.storage,
-			#[cfg(feature = "std")]
 			recorder: self.recorder,
 		}
 	}

 	/// Use the given `cache` for the to be configured [`TrieBackend`].
-	pub fn with_cache<LC>(self, cache: LC) -> TrieBackendBuilder<S, H, LC> {
+	pub fn with_cache<LC>(self, cache: LC) -> TrieBackendBuilder<S, H, LC, R> {
 		TrieBackendBuilder {
 			cache: Some(cache),
 			root: self.root,
 			storage: self.storage,
-			#[cfg(feature = "std")]
 			recorder: self.recorder,
 		}
 	}

 	/// Build the configured [`TrieBackend`].
-	#[cfg(feature = "std")]
-	pub fn build(self) -> TrieBackend<S, H, C> {
+	pub fn build(self) -> TrieBackend<S, H, C, R> {
 		TrieBackend {
 			essence: TrieBackendEssence::new_with_cache_and_recorder(
 				self.storage,
@@ -267,27 +290,18 @@ where
 			next_storage_key_cache: Default::default(),
 		}
 	}
-
-	/// Build the configured [`TrieBackend`].
-	#[cfg(not(feature = "std"))]
-	pub fn build(self) -> TrieBackend<S, H, C> {
-		TrieBackend {
-			essence: TrieBackendEssence::new_with_cache(self.storage, self.root, self.cache),
-			next_storage_key_cache: Default::default(),
-		}
-	}
 }

 /// A cached iterator.
-struct CachedIter<S, H, C>
+struct CachedIter<S, H, C, R>
 where
 	H: Hasher,
 {
 	last_key: sp_std::vec::Vec<u8>,
-	iter: RawIter<S, H, C>,
+	iter: RawIter<S, H, C, R>,
 }

-impl<S, H, C> Default for CachedIter<S, H, C>
+impl<S, H, C, R> Default for CachedIter<S, H, C, R>
 where
 	H: Hasher,
 {
@@ -313,23 +327,32 @@ fn access_cache<T, R>(cell: &CacheCell<T>, callback: impl FnOnce(&mut T) -> R) -
 }

 /// Patricia trie-based backend. Transaction type is an overlay of changes to commit.
-pub struct TrieBackend<S: TrieBackendStorage<H>, H: Hasher, C = DefaultCache<H>> {
-	pub(crate) essence: TrieBackendEssence<S, H, C>,
-	next_storage_key_cache: CacheCell<Option<CachedIter<S, H, C>>>,
+pub struct TrieBackend<
+	S: TrieBackendStorage<H>,
+	H: Hasher,
+	C = DefaultCache<H>,
+	R = DefaultRecorder<H>,
+> {
+	pub(crate) essence: TrieBackendEssence<S, H, C, R>,
+	next_storage_key_cache: CacheCell<Option<CachedIter<S, H, C, R>>>,
 }

-impl<S: TrieBackendStorage<H>, H: Hasher, C: TrieCacheProvider<H> + Send + Sync>
-	TrieBackend<S, H, C>
+impl<
+		S: TrieBackendStorage<H>,
+		H: Hasher,
+		C: TrieCacheProvider<H> + Send + Sync,
+		R: TrieRecorderProvider<H> + Send + Sync,
+	> TrieBackend<S, H, C, R>
 where
 	H::Out: Codec,
 {
 	#[cfg(test)]
-	pub(crate) fn from_essence(essence: TrieBackendEssence<S, H, C>) -> Self {
+	pub(crate) fn from_essence(essence: TrieBackendEssence<S, H, C, R>) -> Self {
 		Self { essence, next_storage_key_cache: Default::default() }
 	}

 	/// Get backend essence reference.
-	pub fn essence(&self) -> &TrieBackendEssence<S, H, C> {
+	pub fn essence(&self) -> &TrieBackendEssence<S, H, C, R> {
 		&self.essence
 	}

@@ -361,28 +384,31 @@ where
 	/// Extract the [`StorageProof`].
 	///
 	/// This only returns `Some` when there was a recorder set.
-	#[cfg(feature = "std")]
 	pub fn extract_proof(mut self) -> Option<StorageProof> {
-		self.essence.recorder.take().map(|r| r.drain_storage_proof())
+		self.essence.recorder.take().and_then(|r| r.drain_storage_proof())
 	}
 }

-impl<S: TrieBackendStorage<H>, H: Hasher, C: TrieCacheProvider<H>> sp_std::fmt::Debug
-	for TrieBackend<S, H, C>
+impl<S: TrieBackendStorage<H>, H: Hasher, C: TrieCacheProvider<H>, R: TrieRecorderProvider<H>>
+	sp_std::fmt::Debug for TrieBackend<S, H, C, R>
 {
 	fn fmt(&self, f: &mut sp_std::fmt::Formatter<'_>) -> sp_std::fmt::Result {
 		write!(f, "TrieBackend")
 	}
 }

-impl<S: TrieBackendStorage<H>, H: Hasher, C: TrieCacheProvider<H> + Send + Sync> Backend<H>
-	for TrieBackend<S, H, C>
+impl<
+		S: TrieBackendStorage<H>,
+		H: Hasher,
+		C: TrieCacheProvider<H> + Send + Sync,
+		R: TrieRecorderProvider<H> + Send + Sync,
+	> Backend<H> for TrieBackend<S, H, C, R>
 where
 	H::Out: Ord + Codec,
 {
 	type Error = crate::DefaultError;
 	type TrieBackendStorage = S;
-	type RawIter = crate::trie_backend_essence::RawIter<S, H, C>;
+	type RawIter = crate::trie_backend_essence::RawIter<S, H, C, R>;

 	fn storage_hash(&self, key: &[u8]) -> Result<Option<H::Out>, Self::Error> {
 		self.essence.storage_hash(key)