Light friendly storage tracking: changes trie + extending over ranges (#628)

* changes_trie * changs_trie: continue * changes_trie: adding tests * fixed TODO * removed obsolete ExtrinsicChanges * encodable ChangesTrieConfiguration * removed polkadot fle * fixed grumbles * ext_storage_changes_root returns u32 * moved changes trie root to digest * removed commented code * read storage values from native code * fixed grumbles * fixed grumbles * missing comma
2026-04-28 07:27:55 +00:00 · 2018-09-18 10:14:41 +03:00
parent 24479cd7f5
commit 7fa337afbc
64 changed files with 3130 additions and 788 deletions
@@ -0,0 +1,371 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! The overlayed changes to state.
+
+use std::collections::{HashMap, HashSet};
+use codec::Decode;
+use changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig};
+
+/// The overlayed changes to state to be queried on top of the backend.
+///
+/// A transaction shares all prospective changes within an inner overlay
+/// that can be cleared.
+#[derive(Debug, Default, Clone)]
+pub struct OverlayedChanges {
+	/// Changes that are not yet committed.
+	pub(crate) prospective: HashMap<Vec<u8>, OverlayedValue>,
+	/// Committed changes.
+	pub(crate) committed: HashMap<Vec<u8>, OverlayedValue>,
+	/// Changes trie configuration. None by default, but could be installed by the
+	/// runtime if it supports change tries.
+	pub(crate) changes_trie_config: Option<ChangesTrieConfig>,
+}
+
+/// The storage value, used inside OverlayedChanges.
+#[derive(Debug, Default, Clone)]
+#[cfg_attr(test, derive(PartialEq))]
+pub struct OverlayedValue {
+	/// Current value. None if value has been deleted.
+	pub value: Option<Vec<u8>>,
+	/// The set of extinsic indices where the values has been changed.
+	/// Is filled only if runtime ahs announced changes trie support.
+	pub extrinsics: Option<HashSet<u32>>,
+}
+
+impl OverlayedChanges {
+	/// Sets the changes trie configuration.
+	///
+	/// Returns false if configuration has been set already and we now trying
+	/// to install different configuration. This isn't supported now.
+	#[must_use = "Result must be checked"]
+	pub(crate) fn set_changes_trie_config(&mut self, config: ChangesTrieConfig) -> bool {
+		if let Some(ref old_config) = self.changes_trie_config {
+			// we do not support changes trie configuration' change now
+			if *old_config != config {
+				return false;
+			}
+		}
+
+		self.changes_trie_config = Some(config);
+		true
+	}
+
+	/// Returns a double-Option: None if the key is unknown (i.e. and the query should be refered
+	/// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose
+	/// value has been set.
+	pub fn storage(&self, key: &[u8]) -> Option<Option<&[u8]>> {
+		self.prospective.get(key)
+			.or_else(|| self.committed.get(key))
+			.map(|x| x.value.as_ref().map(AsRef::as_ref))
+	}
+
+	/// Inserts the given key-value pair into the prospective change set.
+	///
+	/// `None` can be used to delete a value specified by the given key.
+	pub(crate) fn set_storage(&mut self, key: Vec<u8>, val: Option<Vec<u8>>) {
+		let extrinsic_index = self.extrinsic_index();
+		let entry = self.prospective.entry(key).or_default();
+		entry.value = val;
+
+		if let Some(extrinsic) = extrinsic_index {
+			entry.extrinsics.get_or_insert_with(Default::default)
+				.insert(extrinsic);
+		}
+	}
+
+	/// Removes all key-value pairs which keys share the given prefix.
+	///
+	/// NOTE that this doesn't take place immediately but written into the prospective
+	/// change set, and still can be reverted by [`discard_prospective`].
+	///
+	/// [`discard_prospective`]: #method.discard_prospective
+	pub(crate) fn clear_prefix(&mut self, prefix: &[u8]) {
+		let extrinsic_index = self.extrinsic_index();
+
+		// Iterate over all prospective and mark all keys that share
+		// the given prefix as removed (None).
+		for (key, entry) in self.prospective.iter_mut() {
+			if key.starts_with(prefix) {
+				entry.value = None;
+
+				if let Some(extrinsic) = extrinsic_index {
+					entry.extrinsics.get_or_insert_with(Default::default)
+						.insert(extrinsic);
+				}
+			}
+		}
+
+		// Then do the same with keys from commited changes.
+		// NOTE that we are making changes in the prospective change set.
+		for key in self.committed.keys() {
+			if key.starts_with(prefix) {
+				let entry = self.prospective.entry(key.clone()).or_default();
+				entry.value = None;
+
+				if let Some(extrinsic) = extrinsic_index {
+					entry.extrinsics.get_or_insert_with(Default::default)
+						.insert(extrinsic);
+				}
+			}
+		}
+	}
+
+	/// Discard prospective changes to state.
+	pub fn discard_prospective(&mut self) {
+		self.prospective.clear();
+	}
+
+	/// Commit prospective changes to state.
+	pub fn commit_prospective(&mut self) {
+		if self.committed.is_empty() {
+			::std::mem::swap(&mut self.prospective, &mut self.committed);
+		} else {
+			for (key, val) in self.prospective.drain() {
+				let entry = self.committed.entry(key).or_default();
+				entry.value = val.value;
+
+				if let Some(prospective_extrinsics) = val.extrinsics {
+					entry.extrinsics.get_or_insert_with(Default::default)
+						.extend(prospective_extrinsics);
+				}
+			}
+		}
+	}
+
+	/// Drain committed changes to an iterator.
+	///
+	/// Panics:
+	/// Will panic if there are any uncommitted prospective changes.
+	pub fn drain<'a>(&'a mut self) -> impl Iterator<Item=(Vec<u8>, OverlayedValue)> + 'a {
+		assert!(self.prospective.is_empty());
+		self.committed.drain()
+	}
+
+	/// Consume `OverlayedChanges` and take committed set.
+	///
+	/// Panics:
+	/// Will panic if there are any uncommitted prospective changes.
+	pub fn into_committed(self) -> impl Iterator<Item=(Vec<u8>, Option<Vec<u8>>)> {
+		assert!(self.prospective.is_empty());
+		self.committed.into_iter().map(|(k, v)| (k, v.value))
+	}
+
+	/// Inserts storage entry responsible for current extrinsic index.
+	#[cfg(test)]
+	pub(crate) fn set_extrinsic_index(&mut self, extrinsic_index: u32) {
+		use codec::Encode;
+		self.prospective.insert(b":extrinsic_index".to_vec(), OverlayedValue {
+			value: Some(extrinsic_index.encode()),
+			extrinsics: None,
+		});
+	}
+
+	/// Returns current extrinsic index to use in changes trie construction.
+	/// None is returned if it is not set or changes trie config is not set.
+	/// Persistent value (from the backend) can be ignored because runtime must
+	/// set this index before first and unset after last extrinsic is executied.
+	/// Changes that are made outside of extrinsics, are marked with
+	/// `NO_EXTRINSIC_INDEX` index.
+	fn extrinsic_index(&self) -> Option<u32> {
+		match self.changes_trie_config.is_some() {
+			true => Some(
+				self.storage(b":extrinsic_index")
+					.and_then(|idx| idx.and_then(|idx| Decode::decode(&mut &*idx)))
+					.unwrap_or(NO_EXTRINSIC_INDEX)),
+			false => None,
+		}
+	}
+}
+
+#[cfg(test)]
+impl From<Option<Vec<u8>>> for OverlayedValue {
+	fn from(value: Option<Vec<u8>>) -> OverlayedValue {
+		OverlayedValue { value, ..Default::default() }
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use primitives::{Blake2Hasher, RlpCodec, H256};
+	use backend::InMemory;
+	use changes_trie::InMemoryStorage as InMemoryChangesTrieStorage;
+	use ext::Ext;
+	use {Externalities};
+	use super::*;
+
+	fn strip_extrinsic_index(map: &HashMap<Vec<u8>, OverlayedValue>) -> HashMap<Vec<u8>, OverlayedValue> {
+		let mut clone = map.clone();
+		clone.remove(&b":extrinsic_index".to_vec());
+		clone
+	}
+
+	#[test]
+	fn overlayed_storage_works() {
+		let mut overlayed = OverlayedChanges::default();
+
+		let key = vec![42, 69, 169, 142];
+
+		assert!(overlayed.storage(&key).is_none());
+
+		overlayed.set_storage(key.clone(), Some(vec![1, 2, 3]));
+		assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..]));
+
+		overlayed.commit_prospective();
+		assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..]));
+
+		overlayed.set_storage(key.clone(), Some(vec![]));
+		assert_eq!(overlayed.storage(&key).unwrap(), Some(&[][..]));
+
+		overlayed.set_storage(key.clone(), None);
+		assert!(overlayed.storage(&key).unwrap().is_none());
+
+		overlayed.discard_prospective();
+		assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..]));
+
+		overlayed.set_storage(key.clone(), None);
+		overlayed.commit_prospective();
+		assert!(overlayed.storage(&key).unwrap().is_none());
+	}
+
+	#[test]
+	fn overlayed_storage_root_works() {
+		let initial: HashMap<_, _> = vec![
+			(b"doe".to_vec(), b"reindeer".to_vec()),
+			(b"dog".to_vec(), b"puppyXXX".to_vec()),
+			(b"dogglesworth".to_vec(), b"catXXX".to_vec()),
+			(b"doug".to_vec(), b"notadog".to_vec()),
+		].into_iter().collect();
+		let backend = InMemory::<Blake2Hasher, RlpCodec>::from(initial);
+		let mut overlay = OverlayedChanges {
+			committed: vec![
+				(b"dog".to_vec(), Some(b"puppy".to_vec()).into()),
+				(b"dogglesworth".to_vec(), Some(b"catYYY".to_vec()).into()),
+				(b"doug".to_vec(), Some(vec![]).into()),
+			].into_iter().collect(),
+			prospective: vec![
+				(b"dogglesworth".to_vec(), Some(b"cat".to_vec()).into()),
+				(b"doug".to_vec(), None.into()),
+			].into_iter().collect(),
+			..Default::default()
+		};
+
+		let changes_trie_storage = InMemoryChangesTrieStorage::new();
+		let mut ext = Ext::new(&mut overlay, &backend, Some(&changes_trie_storage));
+		const ROOT: [u8; 32] = hex!("6ca394ff9b13d6690a51dea30b1b5c43108e52944d30b9095227c49bae03ff8b");
+		assert_eq!(ext.storage_root(), H256(ROOT));
+	}
+
+	#[test]
+	fn changes_trie_configuration_is_saved() {
+		let mut overlay = OverlayedChanges::default();
+		assert!(overlay.changes_trie_config.is_none());
+		assert_eq!(overlay.set_changes_trie_config(ChangesTrieConfig {
+			digest_interval: 4, digest_levels: 1,
+		}), true);
+		assert!(overlay.changes_trie_config.is_some());
+	}
+
+	#[test]
+	fn changes_trie_configuration_is_saved_twice() {
+		let mut overlay = OverlayedChanges::default();
+		assert!(overlay.changes_trie_config.is_none());
+		assert_eq!(overlay.set_changes_trie_config(ChangesTrieConfig {
+			digest_interval: 4, digest_levels: 1,
+		}), true);
+		overlay.set_extrinsic_index(0);
+		overlay.set_storage(vec![1], Some(vec![2]));
+		assert_eq!(overlay.set_changes_trie_config(ChangesTrieConfig {
+			digest_interval: 4, digest_levels: 1,
+		}), true);
+		assert_eq!(
+			strip_extrinsic_index(&overlay.prospective),
+			vec![
+				(vec![1], OverlayedValue { value: Some(vec![2]), extrinsics: Some(vec![0].into_iter().collect()) }),
+			].into_iter().collect(),
+		);
+	}
+
+	#[test]
+	fn panics_when_trying_to_save_different_changes_trie_configuration() {
+		let mut overlay = OverlayedChanges::default();
+		assert_eq!(overlay.set_changes_trie_config(ChangesTrieConfig {
+			digest_interval: 4, digest_levels: 1,
+		}), true);
+		assert_eq!(overlay.set_changes_trie_config(ChangesTrieConfig {
+			digest_interval: 2, digest_levels: 1,
+		}), false);
+	}
+
+	#[test]
+	fn extrinsic_changes_are_collected() {
+		let mut overlay = OverlayedChanges::default();
+		let _ = overlay.set_changes_trie_config(ChangesTrieConfig {
+			digest_interval: 4, digest_levels: 1,
+		});
+
+		overlay.set_storage(vec![100], Some(vec![101]));
+
+		overlay.set_extrinsic_index(0);
+		overlay.set_storage(vec![1], Some(vec![2]));
+
+		overlay.set_extrinsic_index(1);
+		overlay.set_storage(vec![3], Some(vec![4]));
+
+		overlay.set_extrinsic_index(2);
+		overlay.set_storage(vec![1], Some(vec![6]));
+
+		assert_eq!(strip_extrinsic_index(&overlay.prospective),
+			vec![
+				(vec![1], OverlayedValue { value: Some(vec![6]), extrinsics: Some(vec![0, 2].into_iter().collect()) }),
+				(vec![3], OverlayedValue { value: Some(vec![4]), extrinsics: Some(vec![1].into_iter().collect()) }),
+				(vec![100], OverlayedValue { value: Some(vec![101]), extrinsics: Some(vec![NO_EXTRINSIC_INDEX].into_iter().collect()) }),
+			].into_iter().collect());
+
+		overlay.commit_prospective();
+
+		overlay.set_extrinsic_index(3);
+		overlay.set_storage(vec![3], Some(vec![7]));
+
+		overlay.set_extrinsic_index(4);
+		overlay.set_storage(vec![1], Some(vec![8]));
+
+		assert_eq!(strip_extrinsic_index(&overlay.committed),
+			vec![
+				(vec![1], OverlayedValue { value: Some(vec![6]), extrinsics: Some(vec![0, 2].into_iter().collect()) }),
+				(vec![3], OverlayedValue { value: Some(vec![4]), extrinsics: Some(vec![1].into_iter().collect()) }),
+				(vec![100], OverlayedValue { value: Some(vec![101]), extrinsics: Some(vec![NO_EXTRINSIC_INDEX].into_iter().collect()) }),
+			].into_iter().collect());
+
+		assert_eq!(strip_extrinsic_index(&overlay.prospective),
+			vec![
+				(vec![1], OverlayedValue { value: Some(vec![8]), extrinsics: Some(vec![4].into_iter().collect()) }),
+				(vec![3], OverlayedValue { value: Some(vec![7]), extrinsics: Some(vec![3].into_iter().collect()) }),
+			].into_iter().collect());
+
+		overlay.commit_prospective();
+
+		assert_eq!(strip_extrinsic_index(&overlay.committed),
+			vec![
+				(vec![1], OverlayedValue { value: Some(vec![8]), extrinsics: Some(vec![0, 2, 4].into_iter().collect()) }),
+				(vec![3], OverlayedValue { value: Some(vec![7]), extrinsics: Some(vec![1, 3].into_iter().collect()) }),
+				(vec![100], OverlayedValue { value: Some(vec![101]), extrinsics: Some(vec![NO_EXTRINSIC_INDEX].into_iter().collect()) }),
+			].into_iter().collect());
+
+		assert_eq!(overlay.prospective,
+			Default::default());
+	}
+}