Make it possible to calculate the storage root as often as you want (#7714)

* Make it possible to calculate the storage as often as you want

So, until now each Substrate based blockchain has calculated the storage
root once, at the end of the block. Now there is Frontier that wants to
calculate some intermediate storage root. However this failed on block
import. The problem with that was the extrinsics root. When building the
block we stored `Default::default()` as extrinsics root, because yeah,
we don't know the extrinsics root before finishing the block. At the end
this extrinsics root was then calculated. But on block import we passed
the already known extrinsics root. This was no problem, as we removed
this value at the end of the block. However when you all the storage
root in between, that changes the storage root between block building
and block import.

This pr changes this behavior. It removes the `ExtrinsicsRoot` storage
entry and also doesn't pass it anymore to `System::initialize`. By doing
it, we remove the difference in the storage and fix the storage root mismatch.

* Fix bug with incorrectly calculating the extrinscs root

* Review feedback

substrate/frame/system/src/lib.rs

@@ -107,7 +107,7 @@ use sp_runtime::{
 		self, CheckEqual, AtLeast32Bit, Zero, Lookup, LookupError,
 		SimpleBitOps, Hash, Member, MaybeDisplay, BadOrigin,
 		MaybeSerialize, MaybeSerializeDeserialize, MaybeMallocSizeOf, StaticLookup, One, Bounded,
-		Dispatchable, AtLeast32BitUnsigned
+		Dispatchable, AtLeast32BitUnsigned, Saturating,
 	},
 	offchain::storage_lock::BlockNumberProvider,
 };
@@ -405,9 +405,6 @@ decl_storage! {
 		/// Hash of the previous block.
 		ParentHash get(fn parent_hash) build(|_| hash69()): T::Hash;
 
-		/// Extrinsics root of the current block, also part of the block header.
-		ExtrinsicsRoot get(fn extrinsics_root): T::Hash;
-
 		/// Digest of the current block, also part of the block header.
 		Digest get(fn digest): DigestOf<T>;
 
@@ -989,7 +986,6 @@ impl<T: Config> Module<T> {
 	pub fn initialize(
 		number: &T::BlockNumber,
 		parent_hash: &T::Hash,
-		txs_root: &T::Hash,
 		digest: &DigestOf<T>,
 		kind: InitKind,
 	) {
@@ -1000,7 +996,6 @@ impl<T: Config> Module<T> {
 		<Digest<T>>::put(digest);
 		<ParentHash<T>>::put(parent_hash);
 		<BlockHash<T>>::insert(*number - One::one(), parent_hash);
-		<ExtrinsicsRoot<T>>::put(txs_root);
 
 		// Remove previous block data from storage
 		BlockWeight::kill();
@@ -1017,7 +1012,6 @@ impl<T: Config> Module<T> {
 	/// resulting header for this block.
 	pub fn finalize() -> T::Header {
 		ExecutionPhase::kill();
-		ExtrinsicCount::kill();
 		AllExtrinsicsLen::kill();
 
 		// The following fields
@@ -1034,17 +1028,18 @@ impl<T: Config> Module<T> {
 		let parent_hash = <ParentHash<T>>::get();
 		let mut digest = <Digest<T>>::get();
 
-		let extrinsics_root = <ExtrinsicsRoot<T>>::take();
+		let extrinsics = (0..ExtrinsicCount::take().unwrap_or_default())
+			.map(ExtrinsicData::take)
+			.collect();
+		let extrinsics_root = extrinsics_data_root::<T::Hashing>(extrinsics);
 
 		// move block hash pruning window by one block
-		let block_hash_count = <T::BlockHashCount>::get();
-		if number > block_hash_count {
-			let to_remove = number - block_hash_count - One::one();
+		let block_hash_count = T::BlockHashCount::get();
+		let to_remove = number.saturating_sub(block_hash_count).saturating_sub(One::one());
 
-			// keep genesis hash
-			if to_remove != Zero::zero() {
-				<BlockHash<T>>::remove(to_remove);
-			}
+		// keep genesis hash
+		if !to_remove.is_zero() {
+			<BlockHash<T>>::remove(to_remove);
 		}
 
 		let storage_root = T::Hash::decode(&mut &sp_io::storage::root()[..])
@@ -1138,12 +1133,10 @@ impl<T: Config> Module<T> {
 		Account::<T>::mutate(who, |a| a.nonce += T::Index::one());
 	}
 
-	/// Note what the extrinsic data of the current extrinsic index is. If this
-	/// is called, then ensure `derive_extrinsics` is also called before
-	/// block-building is completed.
+	/// Note what the extrinsic data of the current extrinsic index is.
 	///
-	/// NOTE: This function is called only when the block is being constructed locally.
-	/// `execute_block` doesn't note any extrinsics.
+	/// This is required to be called before applying an extrinsic. The data will used
+	/// in [`Self::finalize`] to calculate the correct extrinsics root.
 	pub fn note_extrinsic(encoded_xt: Vec<u8>) {
 		ExtrinsicData::insert(Self::extrinsic_index().unwrap_or_default(), encoded_xt);
 	}
@@ -1182,14 +1175,6 @@ impl<T: Config> Module<T> {
 		ExecutionPhase::put(Phase::ApplyExtrinsic(0))
 	}
 
-	/// Remove all extrinsic data and save the extrinsics trie root.
-	pub fn derive_extrinsics() {
-		let extrinsics = (0..ExtrinsicCount::get().unwrap_or_default())
-			.map(ExtrinsicData::take).collect();
-		let xts_root = extrinsics_data_root::<T::Hashing>(extrinsics);
-		<ExtrinsicsRoot<T>>::put(xts_root);
-	}
-
 	/// An account is being created.
 	pub fn on_created_account(who: T::AccountId) {
 		T::OnNewAccount::on_new_account(&who);

substrate/frame/system/src/tests.rs

@@ -18,7 +18,7 @@
 use crate::*;
 use mock::{*, Origin};
 use sp_core::H256;
-use sp_runtime::DispatchError;
+use sp_runtime::{DispatchError, traits::{Header, BlakeTwo256}};
 use frame_support::weights::WithPostDispatchInfo;
 
 #[test]
@@ -55,7 +55,6 @@ fn deposit_event_should_work() {
 		System::initialize(
 			&1,
 			&[0u8; 32].into(),
-			&[0u8; 32].into(),
 			&Default::default(),
 			InitKind::Full,
 		);
@@ -76,7 +75,6 @@ fn deposit_event_should_work() {
 		System::initialize(
 			&2,
 			&[0u8; 32].into(),
-			&[0u8; 32].into(),
 			&Default::default(),
 			InitKind::Full,
 		);
@@ -133,7 +131,6 @@ fn deposit_event_uses_actual_weight() {
 		System::initialize(
 			&1,
 			&[0u8; 32].into(),
-			&[0u8; 32].into(),
 			&Default::default(),
 			InitKind::Full,
 		);
@@ -218,7 +215,6 @@ fn deposit_event_topics() {
 		System::initialize(
 			&BLOCK_NUMBER,
 			&[0u8; 32].into(),
-			&[0u8; 32].into(),
 			&Default::default(),
 			InitKind::Full,
 		);
@@ -284,7 +280,6 @@ fn prunes_block_hash_mappings() {
 			System::initialize(
 				&n,
 				&[n as u8 - 1; 32].into(),
-				&[0u8; 32].into(),
 				&Default::default(),
 				InitKind::Full,
 			);
@@ -422,3 +417,28 @@ fn ensure_one_of_works() {
 	assert_eq!(ensure_root_or_signed(RawOrigin::Signed(0)).unwrap(), Either::Right(0));
 	assert!(ensure_root_or_signed(RawOrigin::None).is_err())
 }
+
+#[test]
+fn extrinsics_root_is_calculated_correctly() {
+	new_test_ext().execute_with(|| {
+		System::initialize(
+			&1,
+			&[0u8; 32].into(),
+			&Default::default(),
+			InitKind::Full,
+		);
+		System::note_finished_initialize();
+		System::note_extrinsic(vec![1]);
+		System::note_applied_extrinsic(&Ok(().into()), Default::default());
+		System::note_extrinsic(vec![2]);
+		System::note_applied_extrinsic(
+			&Err(DispatchError::BadOrigin.into()),
+			Default::default()
+		);
+		System::note_finished_extrinsics();
+		let header = System::finalize();
+
+		let ext_root = extrinsics_data_root::<BlakeTwo256>(vec![vec![1], vec![2]]);
+		assert_eq!(ext_root, *header.extrinsics_root());
+	});
+}