diff --git a/cumulus/Cargo.lock b/cumulus/Cargo.lock index 95e39bad8f..5054b73f82 100644 --- a/cumulus/Cargo.lock +++ b/cumulus/Cargo.lock @@ -1193,9 +1193,11 @@ dependencies = [ "sp-blockchain", "sp-consensus", "sp-core", + "sp-externalities", "sp-io", "sp-keyring", "sp-runtime", + "sp-state-machine", "sp-std", "sp-trie", "trie-db", diff --git a/cumulus/runtime/Cargo.toml b/cumulus/runtime/Cargo.toml index 19a4dace6c..b781567896 100644 --- a/cumulus/runtime/Cargo.toml +++ b/cumulus/runtime/Cargo.toml @@ -21,6 +21,8 @@ sp-core = { git = "https://github.com/paritytech/substrate", default-features = sp-io = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" } frame-executive = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" } sp-trie = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" } +sp-state-machine = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" } +sp-externalities = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" } # Polkadot dependencies parachain = { package = "polkadot-parachain", git = "https://github.com/paritytech/polkadot", default-features = false, features = [ "wasm-api" ] , branch = "master" } @@ -48,6 +50,8 @@ std = [ "memory-db/std", "hash-db/std", "trie-db/std", + "sp-state-machine/std", + "sp-externalities/std", "sp-trie/std", "parachain/std", ] diff --git a/cumulus/runtime/src/validate_block/implementation.rs b/cumulus/runtime/src/validate_block/implementation.rs index abcbea5ffd..51bc9c3619 100644 --- a/cumulus/runtime/src/validate_block/implementation.rs +++ b/cumulus/runtime/src/validate_block/implementation.rs @@ -17,18 +17,15 @@ //! The actual implementation of the validate block functionality. use frame_executive::ExecuteBlock; -use sp_runtime::traits::{Block as BlockT, HashFor, Header as HeaderT}; +use sp_runtime::traits::{Block as BlockT, HashFor, NumberFor, Header as HeaderT}; -use sp_std::{boxed::Box, collections::btree_map::BTreeMap, ops::Bound, vec::Vec}; -use sp_trie::{delta_trie_root, read_trie_value, Layout, MemoryDB, StorageProof}; +use sp_std::{boxed::Box, vec::Vec}; use hash_db::{HashDB, EMPTY_PREFIX}; -use trie_db::{Trie, TrieDB, TrieDBIterator}; - use parachain::primitives::{HeadData, ValidationCode, ValidationParams, ValidationResult}; -use codec::{Decode, Encode, EncodeAppend}; +use codec::{Decode, Encode}; use cumulus_primitives::{ well_known_keys::{ @@ -36,60 +33,25 @@ use cumulus_primitives::{ }, GenericUpwardMessage, ValidationData, }; +use sp_externalities::{set_and_run_with_externalities}; +use sp_externalities::{Externalities, ExtensionStore, Error, Extension}; +use sp_trie::MemoryDB; +use sp_std::{any::{TypeId, Any}}; +use sp_core::storage::{ChildInfo, TrackedStorageKey}; -/// Stores the global [`Storage`] instance. -/// -/// As wasm is always executed with one thread, this global varibale is safe! -static mut STORAGE: Option> = None; +type StorageValue = Vec; +type StorageKey = Vec; -/// Runs the given `call` with the global storage and returns the result of the call. -/// -/// # Panic -/// -/// Panics if the [`STORAGE`] is not initialized. -fn with_storage(call: impl FnOnce(&mut dyn Storage) -> R) -> R { - let mut storage = unsafe { - STORAGE - .take() - .expect("`STORAGE` needs to be set before calling this function.") - }; +type Ext<'a, B: BlockT> = sp_state_machine::Ext< + 'a, + HashFor, + NumberFor, + sp_state_machine::TrieBackend>, HashFor>, +>; - let res = call(&mut *storage); - - unsafe { - STORAGE = Some(storage); - } - - res -} - -/// Abstract the storage into a trait without `Block` generic. -trait Storage { - /// Retrieve the value for the given key. - fn get(&self, key: &[u8]) -> Option>; - - /// Retrieve the value for the given key only if modified. - fn modified(&self, key: &[u8]) -> Option>; - - /// Insert the given key and value. - fn insert(&mut self, key: &[u8], value: &[u8]); - - /// Remove key and value. - fn remove(&mut self, key: &[u8]); - - /// Calculate the storage root. - /// - /// Returns the SCALE encoded hash. - fn storage_root(&mut self) -> Vec; - - /// Clear all keys that start with the given prefix. - fn clear_prefix(&mut self, prefix: &[u8]); - - /// Append the value to the given key - fn storage_append(&mut self, key: &[u8], value: Vec); - - /// Get the next storage key after the given `key`. - fn next_key(&self, key: &[u8]) -> Option>; +fn with_externalities R, R>(f: F) -> R { + sp_externalities::with_externalities(f) + .expect("Environmental externalities not set.") } /// Implement `Encode` by forwarding the stored raw vec. @@ -118,15 +80,23 @@ pub fn validate_block>(params: ValidationParams) - "Invalid parent hash", ); - let storage_inner = WitnessStorage::::new( - block_data.storage_proof, - parent_head.state_root().clone(), - params, - ) - .expect("Witness data and storage root always match; qed"); + let db = block_data.storage_proof.into_memory_db(); + let root = parent_head.state_root().clone(); + if !HashDB::, _>::contains(&db, &root, EMPTY_PREFIX) { + panic!("Witness data does not contain given storage root."); + } + let backend = sp_state_machine::TrieBackend::new( + db, + root, + ); + let mut overlay = sp_state_machine::OverlayedChanges::default(); + let mut cache = Default::default(); + let mut ext = WitnessExt:: { + inner: Ext::::new(&mut overlay, &mut cache, &backend), + params: ¶ms, + }; let _guard = unsafe { - STORAGE = Some(Box::new(storage_inner)); ( // Replace storage calls with our own implementations sp_io::storage::host_read.replace_implementation(host_storage_read), @@ -139,30 +109,56 @@ pub fn validate_block>(params: ValidationParams) - sp_io::storage::host_changes_root.replace_implementation(host_storage_changes_root), sp_io::storage::host_append.replace_implementation(host_storage_append), sp_io::storage::host_next_key.replace_implementation(host_storage_next_key), + sp_io::storage::host_start_transaction.replace_implementation(host_storage_start_transaction), + sp_io::storage::host_rollback_transaction.replace_implementation( + host_storage_rollback_transaction + ), + sp_io::storage::host_commit_transaction.replace_implementation( + host_storage_commit_transaction + ), + sp_io::default_child_storage::host_get.replace_implementation(host_default_child_storage_get), + sp_io::default_child_storage::host_read.replace_implementation(host_default_child_storage_read), + sp_io::default_child_storage::host_set.replace_implementation(host_default_child_storage_set), + sp_io::default_child_storage::host_clear.replace_implementation( + host_default_child_storage_clear + ), + sp_io::default_child_storage::host_storage_kill.replace_implementation( + host_default_child_storage_storage_kill + ), + sp_io::default_child_storage::host_exists.replace_implementation( + host_default_child_storage_exists + ), + sp_io::default_child_storage::host_clear_prefix.replace_implementation( + host_default_child_storage_clear_prefix + ), + sp_io::default_child_storage::host_root.replace_implementation(host_default_child_storage_root), + sp_io::default_child_storage::host_next_key.replace_implementation(host_default_child_storage_next_key), ) }; - E::execute_block(block); + set_and_run_with_externalities(&mut ext, || { + E::execute_block(block); + }); // If in the course of block execution new validation code was set, insert // its scheduled upgrade so we can validate that block number later. - let new_validation_code = - with_storage(|storage| storage.modified(NEW_VALIDATION_CODE)).map(ValidationCode); + let new_validation_code = overlay.storage(NEW_VALIDATION_CODE).flatten() + .map(|slice| slice.to_vec()) + .map(ValidationCode); // Extract potential upward messages from the storage. - let upward_messages = match with_storage(|storage| storage.modified(UPWARD_MESSAGES)) { + let upward_messages = match overlay.storage(UPWARD_MESSAGES).flatten() { Some(encoded) => Vec::::decode(&mut &encoded[..]) .expect("Upward messages vec is not correctly encoded in the storage!"), None => Vec::new(), }; - let processed_downward_messages = - with_storage(|storage| storage.modified(PROCESSED_DOWNWARD_MESSAGES)) - .and_then(|v| Decode::decode(&mut &v[..]).ok()) - .unwrap_or_default(); + let processed_downward_messages = overlay.storage(PROCESSED_DOWNWARD_MESSAGES) + .flatten() + .and_then(|v| Decode::decode(&mut &v[..]).ok()) + .unwrap_or_default(); - let validation_data: ValidationData = - with_storage(|storage| storage.modified(VALIDATION_DATA)) + let validation_data: ValidationData = overlay.storage(VALIDATION_DATA).flatten() .and_then(|v| Decode::decode(&mut &v[..]).ok()) .expect("`ValidationData` is required to be placed into the storage!"); @@ -180,192 +176,212 @@ pub fn validate_block>(params: ValidationParams) - /// The storage implementation used when validating a block that is using the /// witness data as source. -struct WitnessStorage { - witness_data: MemoryDB>, - overlay: BTreeMap, Option>>, - storage_root: B::Hash, - validation_params: ValidationParams, +struct WitnessExt<'a, B: BlockT> { + inner: Ext<'a, B>, + params: &'a ValidationParams, } -impl WitnessStorage { - /// Initialize from the given witness data and storage root. - /// - /// Returns an error if given storage root was not found in the witness data. - fn new( - storage_proof: StorageProof, - storage_root: B::Hash, - validation_params: ValidationParams, - ) -> Result { - let db = storage_proof.into_memory_db(); - - if !HashDB::contains(&db, &storage_root, EMPTY_PREFIX) { - return Err("Witness data does not contain given storage root."); - } - - Ok(Self { - witness_data: db, - overlay: Default::default(), - storage_root, - validation_params, - }) - } - - /// Find the next storage key after the given `key` in the trie. - fn trie_next_key(&self, key: &[u8]) -> Option> { - let trie = TrieDB::>>::new(&self.witness_data, &self.storage_root) - .expect("Creates next storage key `TrieDB`"); - let mut iter = trie.iter().expect("Creates trie iterator"); - - // The key just after the one given in input, basically `key++0`. - // Note: We are sure this is the next key if: - // * size of key has no limit (i.e. we can always add 0 to the path), - // * and no keys can be inserted between `key` and `key++0` (this is ensured by sp-io). - let mut potential_next_key = Vec::with_capacity(key.len() + 1); - potential_next_key.extend_from_slice(key); - potential_next_key.push(0); - - iter.seek(&potential_next_key).expect("Seek trie iterator"); - - let next_element = iter.next(); - - if let Some(next_element) = next_element { - let (next_key, _) = next_element.expect("Extracts next key"); - Some(next_key) - } else { - None - } - } - - /// Find the next storage key after the given `key` in the overlay. - fn overlay_next_key(&self, key: &[u8]) -> Option<(&[u8], Option<&[u8]>)> { - let range = (Bound::Excluded(key), Bound::Unbounded); - self.overlay - .range::<[u8], _>(range) - .next() - .map(|(k, v)| (&k[..], v.as_deref())) - } - +impl<'a, B: BlockT> WitnessExt<'a, B> { /// Checks that the encoded `ValidationData` in `data` is correct. /// /// Should be removed with: https://github.com/paritytech/cumulus/issues/217 + /// When removed `WitnessExt` could also be removed. fn check_validation_data(&self, mut data: &[u8]) { let validation_data = ValidationData::decode(&mut data).expect("Invalid `ValidationData`"); assert_eq!( - self.validation_params.parent_head, + self.params.parent_head, validation_data.persisted.parent_head ); assert_eq!( - self.validation_params.relay_chain_height, + self.params.relay_chain_height, validation_data.persisted.block_number ); assert_eq!( - self.validation_params.hrmp_mqc_heads, + self.params.hrmp_mqc_heads, validation_data.persisted.hrmp_mqc_heads ); } } -impl Storage for WitnessStorage { - fn modified(&self, key: &[u8]) -> Option> { - self.overlay.get(key).cloned().unwrap_or(None) +impl<'a, B: BlockT> Externalities for WitnessExt<'a, B> { + fn storage(&self, key: &[u8]) -> Option { + self.inner.storage(key) } - fn get(&self, key: &[u8]) -> Option> { - self.overlay.get(key).cloned().unwrap_or_else(|| { - read_trie_value::>, _>(&self.witness_data, &self.storage_root, key) - .expect("Reading key from trie.") - }) + fn set_offchain_storage(&mut self, key: &[u8], value: Option<&[u8]>) { + self.inner.set_offchain_storage(key, value) } - fn insert(&mut self, key: &[u8], value: &[u8]) { - if key == VALIDATION_DATA { - self.check_validation_data(value); + fn storage_hash(&self, key: &[u8]) -> Option> { + self.inner.storage_hash(key) + } + + fn child_storage( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Option { + self.inner.child_storage(child_info, key) + } + + fn child_storage_hash( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Option> { + self.inner.child_storage_hash(child_info, key) + } + + fn exists_storage(&self, key: &[u8]) -> bool { + self.inner.exists_storage(key) + } + + fn exists_child_storage( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> bool { + self.inner.exists_child_storage(child_info, key) + } + + fn next_storage_key(&self, key: &[u8]) -> Option { + self.inner.next_storage_key(key) + } + + fn next_child_storage_key( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Option { + self.inner.next_child_storage_key(child_info, key) + } + + fn place_storage(&mut self, key: StorageKey, value: Option) { + if let Some(value) = value.as_ref() { + if key == VALIDATION_DATA { + self.check_validation_data(value); + } } - self.overlay.insert(key.to_vec(), Some(value.to_vec())); + self.inner.place_storage(key, value) } - fn remove(&mut self, key: &[u8]) { - self.overlay.insert(key.to_vec(), None); + fn place_child_storage( + &mut self, + child_info: &ChildInfo, + key: StorageKey, + value: Option, + ) { + self.inner.place_child_storage(child_info, key, value) } - fn storage_root(&mut self) -> Vec { - let root = delta_trie_root::>, _, _, _, _, _>( - &mut self.witness_data, - self.storage_root.clone(), - self.overlay - .iter() - .map(|(k, v)| (k.as_ref(), v.as_ref().map(|v| v.as_ref()))), - ) - .expect("Calculates storage root"); - - root.encode() + fn kill_child_storage( + &mut self, + child_info: &ChildInfo, + ) { + self.inner.kill_child_storage(child_info) } fn clear_prefix(&mut self, prefix: &[u8]) { - self.overlay.iter_mut().for_each(|(k, v)| { - if k.starts_with(prefix) { - *v = None; - } - }); - - let trie = match TrieDB::>>::new(&self.witness_data, &self.storage_root) { - Ok(r) => r, - Err(_) => panic!(), - }; - - for x in TrieDBIterator::new_prefixed(&trie, prefix).expect("Creates trie iterator") { - if let Ok((key, _)) = x { - self.overlay.insert(key, None); - } - } + self.inner.clear_prefix(prefix) } - fn storage_append(&mut self, key: &[u8], value: Vec) { - let value_vec = sp_std::vec![EncodeOpaqueValue(value)]; - - let overlay = &mut self.overlay; - let witness_data = &self.witness_data; - let storage_root = &self.storage_root; - - let current_value = overlay.entry(key.to_vec()).or_insert_with(|| { - read_trie_value::>, _>(witness_data, storage_root, key) - .ok() - .flatten() - }); - - let item = current_value.take().unwrap_or_default(); - *current_value = Some( - match Vec::::append_or_new(item, &value_vec) { - Ok(item) => item, - Err(_) => value_vec.encode(), - }, - ); + fn clear_child_prefix( + &mut self, + child_info: &ChildInfo, + prefix: &[u8], + ) { + self.inner.clear_child_prefix(child_info, prefix) } - fn next_key(&self, key: &[u8]) -> Option> { - let next_trie_key = self.trie_next_key(key); - let next_overlay_key = self.overlay_next_key(key); + fn storage_append( + &mut self, + key: Vec, + value: Vec, + ) { + self.inner.storage_append(key, value) + } - match (next_trie_key, next_overlay_key) { - (Some(backend_key), Some(overlay_key)) if &backend_key[..] < overlay_key.0 => { - Some(backend_key) - } - (backend_key, None) => backend_key, - (_, Some(overlay_key)) => { - if overlay_key.1.is_some() { - Some(overlay_key.0.to_vec()) - } else { - self.next_key(&overlay_key.0) - } - } - } + fn chain_id(&self) -> u64 { + 42 + } + + fn storage_root(&mut self) -> Vec { + self.inner.storage_root() + } + + fn child_storage_root( + &mut self, + child_info: &ChildInfo, + ) -> Vec { + self.inner.child_storage_root(child_info) + } + + fn storage_changes_root(&mut self, parent_hash: &[u8]) -> Result>, ()> { + self.inner.storage_changes_root(parent_hash) + } + + fn storage_start_transaction(&mut self) { + self.inner.storage_start_transaction() + } + + fn storage_rollback_transaction(&mut self) -> Result<(), ()> { + self.inner.storage_rollback_transaction() + } + + fn storage_commit_transaction(&mut self) -> Result<(), ()> { + self.inner.storage_commit_transaction() + } + + fn wipe(&mut self) { + self.inner.wipe() + } + + fn commit(&mut self) { + self.inner.commit() + } + + fn read_write_count(&self) -> (u32, u32, u32, u32) { + self.inner.read_write_count() + } + + fn reset_read_write_count(&mut self) { + self.inner.reset_read_write_count() + } + + fn get_whitelist(&self) -> Vec { + self.inner.get_whitelist() + } + + fn set_whitelist(&mut self, new: Vec) { + self.inner.set_whitelist(new) + } +} + +impl<'a, B: BlockT> ExtensionStore for WitnessExt<'a, B> { + fn extension_by_type_id(&mut self, type_id: TypeId) -> Option<&mut dyn Any> { + self.inner.extension_by_type_id(type_id) + } + + fn register_extension_with_type_id( + &mut self, + type_id: TypeId, + extension: Box, + ) -> Result<(), Error> { + self.inner.register_extension_with_type_id(type_id, extension) + } + + fn deregister_extension_by_type_id( + &mut self, + type_id: TypeId, + ) -> Result<(), Error> { + self.inner.deregister_extension_by_type_id(type_id) } } fn host_storage_read(key: &[u8], value_out: &mut [u8], value_offset: u32) -> Option { - match with_storage(|storage| storage.get(key)) { + match with_externalities(|ext| ext.storage(key)) { Some(value) => { let value_offset = value_offset as usize; let data = &value[value_offset.min(value.len())..]; @@ -378,38 +394,130 @@ fn host_storage_read(key: &[u8], value_out: &mut [u8], value_offset: u32) -> Opt } fn host_storage_set(key: &[u8], value: &[u8]) { - with_storage(|storage| storage.insert(key, value)); + with_externalities(|ext| ext.place_storage(key.to_vec(), Some(value.to_vec()))) } fn host_storage_get(key: &[u8]) -> Option> { - with_storage(|storage| storage.get(key).clone()) + with_externalities(|ext| ext.storage(key).clone()) } fn host_storage_exists(key: &[u8]) -> bool { - with_storage(|storage| storage.get(key).is_some()) + with_externalities(|ext| ext.exists_storage(key)) } fn host_storage_clear(key: &[u8]) { - with_storage(|storage| storage.remove(key)); + with_externalities(|ext| ext.place_storage(key.to_vec(), None)) } fn host_storage_root() -> Vec { - with_storage(|storage| storage.storage_root()) + with_externalities(|ext| ext.storage_root()) } fn host_storage_clear_prefix(prefix: &[u8]) { - with_storage(|storage| storage.clear_prefix(prefix)) + with_externalities(|ext| ext.clear_prefix(prefix)) } -fn host_storage_changes_root(_: &[u8]) -> Option> { - // TODO implement it properly - None +fn host_storage_changes_root(parent_hash: &[u8]) -> Option> { + with_externalities(|ext| ext.storage_changes_root(parent_hash).ok().flatten()) } fn host_storage_append(key: &[u8], value: Vec) { - with_storage(|storage| storage.storage_append(key, value)); + with_externalities(|ext| ext.storage_append(key.to_vec(), value)) } fn host_storage_next_key(key: &[u8]) -> Option> { - with_storage(|storage| storage.next_key(key)) + with_externalities(|ext| ext.next_storage_key(key)) +} + +fn host_storage_start_transaction() { + with_externalities(|ext| ext.storage_start_transaction()) +} + +fn host_storage_rollback_transaction() { + with_externalities(|ext| ext.storage_rollback_transaction().ok()) + .expect("No open transaction that can be rolled back."); +} + +fn host_storage_commit_transaction() { + with_externalities(|ext| ext.storage_commit_transaction().ok()) + .expect("No open transaction that can be committed."); +} + +fn host_default_child_storage_get(storage_key: &[u8], key: &[u8]) -> Option> { + let child_info = ChildInfo::new_default(storage_key); + with_externalities(|ext| ext.child_storage(&child_info, key)) +} + +fn host_default_child_storage_read( + storage_key: &[u8], + key: &[u8], + value_out: &mut [u8], + value_offset: u32, +) -> Option { + let child_info = ChildInfo::new_default(storage_key); + match with_externalities(|ext| ext.child_storage(&child_info, key)) { + Some(value) => { + let value_offset = value_offset as usize; + let data = &value[value_offset.min(value.len())..]; + let written = sp_std::cmp::min(data.len(), value_out.len()); + value_out[..written].copy_from_slice(&data[..written]); + Some(value.len() as u32) + } + None => None, + } +} + +fn host_default_child_storage_set( + storage_key: &[u8], + key: &[u8], + value: &[u8], +) { + let child_info = ChildInfo::new_default(storage_key); + with_externalities(|ext| ext.place_child_storage(&child_info, key.to_vec(), Some(value.to_vec()))) +} + +fn host_default_child_storage_clear( + storage_key: &[u8], + key: &[u8], +) { + let child_info = ChildInfo::new_default(storage_key); + with_externalities(|ext| ext.place_child_storage(&child_info, key.to_vec(), None)) +} + +fn host_default_child_storage_storage_kill( + storage_key: &[u8], +) { + let child_info = ChildInfo::new_default(storage_key); + with_externalities(|ext| ext.kill_child_storage(&child_info)) +} + +fn host_default_child_storage_exists( + storage_key: &[u8], + key: &[u8], +) -> bool { + let child_info = ChildInfo::new_default(storage_key); + with_externalities(|ext| ext.exists_child_storage(&child_info, key)) +} + +fn host_default_child_storage_clear_prefix( + storage_key: &[u8], + prefix: &[u8], +) { + let child_info = ChildInfo::new_default(storage_key); + with_externalities(|ext| ext.clear_child_prefix(&child_info, prefix)) +} + +fn host_default_child_storage_root( + storage_key: &[u8], +) -> Vec { + let child_info = ChildInfo::new_default(storage_key); + with_externalities(|ext| ext.child_storage_root(&child_info)) +} + +fn host_default_child_storage_next_key( + storage_key: &[u8], + key: &[u8], +) -> Option> { + let child_info = ChildInfo::new_default(storage_key); + with_externalities(|ext| ext.next_child_storage_key(&child_info, key)) }