Use state machine externalities in validation runtime. (#214)

* test

* switch to branch

* Use modified.

* Use sp_externalities environmental primitives

* Change to latest no_std version, add missing host function boilerplate.

* remove comment (was used for debugging).

* fixes

* with_externalities factor

* Read merged change from overlay.
This commit is contained in:
cheme
2020-11-10 12:23:55 +01:00
committed by GitHub
parent 0a8e1fb9a1
commit 378f6e070b
3 changed files with 343 additions and 229 deletions
@@ -17,18 +17,15 @@
//! The actual implementation of the validate block functionality.
use frame_executive::ExecuteBlock;
use sp_runtime::traits::{Block as BlockT, HashFor, Header as HeaderT};
use sp_runtime::traits::{Block as BlockT, HashFor, NumberFor, Header as HeaderT};
use sp_std::{boxed::Box, collections::btree_map::BTreeMap, ops::Bound, vec::Vec};
use sp_trie::{delta_trie_root, read_trie_value, Layout, MemoryDB, StorageProof};
use sp_std::{boxed::Box, vec::Vec};
use hash_db::{HashDB, EMPTY_PREFIX};
use trie_db::{Trie, TrieDB, TrieDBIterator};
use parachain::primitives::{HeadData, ValidationCode, ValidationParams, ValidationResult};
use codec::{Decode, Encode, EncodeAppend};
use codec::{Decode, Encode};
use cumulus_primitives::{
well_known_keys::{
@@ -36,60 +33,25 @@ use cumulus_primitives::{
},
GenericUpwardMessage, ValidationData,
};
use sp_externalities::{set_and_run_with_externalities};
use sp_externalities::{Externalities, ExtensionStore, Error, Extension};
use sp_trie::MemoryDB;
use sp_std::{any::{TypeId, Any}};
use sp_core::storage::{ChildInfo, TrackedStorageKey};
/// Stores the global [`Storage`] instance.
///
/// As wasm is always executed with one thread, this global varibale is safe!
static mut STORAGE: Option<Box<dyn Storage>> = None;
type StorageValue = Vec<u8>;
type StorageKey = Vec<u8>;
/// Runs the given `call` with the global storage and returns the result of the call.
///
/// # Panic
///
/// Panics if the [`STORAGE`] is not initialized.
fn with_storage<R>(call: impl FnOnce(&mut dyn Storage) -> R) -> R {
let mut storage = unsafe {
STORAGE
.take()
.expect("`STORAGE` needs to be set before calling this function.")
};
type Ext<'a, B: BlockT> = sp_state_machine::Ext<
'a,
HashFor<B>,
NumberFor<B>,
sp_state_machine::TrieBackend<MemoryDB<HashFor<B>>, HashFor<B>>,
>;
let res = call(&mut *storage);
unsafe {
STORAGE = Some(storage);
}
res
}
/// Abstract the storage into a trait without `Block` generic.
trait Storage {
/// Retrieve the value for the given key.
fn get(&self, key: &[u8]) -> Option<Vec<u8>>;
/// Retrieve the value for the given key only if modified.
fn modified(&self, key: &[u8]) -> Option<Vec<u8>>;
/// Insert the given key and value.
fn insert(&mut self, key: &[u8], value: &[u8]);
/// Remove key and value.
fn remove(&mut self, key: &[u8]);
/// Calculate the storage root.
///
/// Returns the SCALE encoded hash.
fn storage_root(&mut self) -> Vec<u8>;
/// Clear all keys that start with the given prefix.
fn clear_prefix(&mut self, prefix: &[u8]);
/// Append the value to the given key
fn storage_append(&mut self, key: &[u8], value: Vec<u8>);
/// Get the next storage key after the given `key`.
fn next_key(&self, key: &[u8]) -> Option<Vec<u8>>;
fn with_externalities<F: FnOnce(&mut dyn Externalities) -> R, R>(f: F) -> R {
sp_externalities::with_externalities(f)
.expect("Environmental externalities not set.")
}
/// Implement `Encode` by forwarding the stored raw vec.
@@ -118,15 +80,23 @@ pub fn validate_block<B: BlockT, E: ExecuteBlock<B>>(params: ValidationParams) -
"Invalid parent hash",
);
let storage_inner = WitnessStorage::<B>::new(
block_data.storage_proof,
parent_head.state_root().clone(),
params,
)
.expect("Witness data and storage root always match; qed");
let db = block_data.storage_proof.into_memory_db();
let root = parent_head.state_root().clone();
if !HashDB::<HashFor<B>, _>::contains(&db, &root, EMPTY_PREFIX) {
panic!("Witness data does not contain given storage root.");
}
let backend = sp_state_machine::TrieBackend::new(
db,
root,
);
let mut overlay = sp_state_machine::OverlayedChanges::default();
let mut cache = Default::default();
let mut ext = WitnessExt::<B> {
inner: Ext::<B>::new(&mut overlay, &mut cache, &backend),
params: &params,
};
let _guard = unsafe {
STORAGE = Some(Box::new(storage_inner));
(
// Replace storage calls with our own implementations
sp_io::storage::host_read.replace_implementation(host_storage_read),
@@ -139,30 +109,56 @@ pub fn validate_block<B: BlockT, E: ExecuteBlock<B>>(params: ValidationParams) -
sp_io::storage::host_changes_root.replace_implementation(host_storage_changes_root),
sp_io::storage::host_append.replace_implementation(host_storage_append),
sp_io::storage::host_next_key.replace_implementation(host_storage_next_key),
sp_io::storage::host_start_transaction.replace_implementation(host_storage_start_transaction),
sp_io::storage::host_rollback_transaction.replace_implementation(
host_storage_rollback_transaction
),
sp_io::storage::host_commit_transaction.replace_implementation(
host_storage_commit_transaction
),
sp_io::default_child_storage::host_get.replace_implementation(host_default_child_storage_get),
sp_io::default_child_storage::host_read.replace_implementation(host_default_child_storage_read),
sp_io::default_child_storage::host_set.replace_implementation(host_default_child_storage_set),
sp_io::default_child_storage::host_clear.replace_implementation(
host_default_child_storage_clear
),
sp_io::default_child_storage::host_storage_kill.replace_implementation(
host_default_child_storage_storage_kill
),
sp_io::default_child_storage::host_exists.replace_implementation(
host_default_child_storage_exists
),
sp_io::default_child_storage::host_clear_prefix.replace_implementation(
host_default_child_storage_clear_prefix
),
sp_io::default_child_storage::host_root.replace_implementation(host_default_child_storage_root),
sp_io::default_child_storage::host_next_key.replace_implementation(host_default_child_storage_next_key),
)
};
E::execute_block(block);
set_and_run_with_externalities(&mut ext, || {
E::execute_block(block);
});
// If in the course of block execution new validation code was set, insert
// its scheduled upgrade so we can validate that block number later.
let new_validation_code =
with_storage(|storage| storage.modified(NEW_VALIDATION_CODE)).map(ValidationCode);
let new_validation_code = overlay.storage(NEW_VALIDATION_CODE).flatten()
.map(|slice| slice.to_vec())
.map(ValidationCode);
// Extract potential upward messages from the storage.
let upward_messages = match with_storage(|storage| storage.modified(UPWARD_MESSAGES)) {
let upward_messages = match overlay.storage(UPWARD_MESSAGES).flatten() {
Some(encoded) => Vec::<GenericUpwardMessage>::decode(&mut &encoded[..])
.expect("Upward messages vec is not correctly encoded in the storage!"),
None => Vec::new(),
};
let processed_downward_messages =
with_storage(|storage| storage.modified(PROCESSED_DOWNWARD_MESSAGES))
.and_then(|v| Decode::decode(&mut &v[..]).ok())
.unwrap_or_default();
let processed_downward_messages = overlay.storage(PROCESSED_DOWNWARD_MESSAGES)
.flatten()
.and_then(|v| Decode::decode(&mut &v[..]).ok())
.unwrap_or_default();
let validation_data: ValidationData =
with_storage(|storage| storage.modified(VALIDATION_DATA))
let validation_data: ValidationData = overlay.storage(VALIDATION_DATA).flatten()
.and_then(|v| Decode::decode(&mut &v[..]).ok())
.expect("`ValidationData` is required to be placed into the storage!");
@@ -180,192 +176,212 @@ pub fn validate_block<B: BlockT, E: ExecuteBlock<B>>(params: ValidationParams) -
/// The storage implementation used when validating a block that is using the
/// witness data as source.
struct WitnessStorage<B: BlockT> {
witness_data: MemoryDB<HashFor<B>>,
overlay: BTreeMap<Vec<u8>, Option<Vec<u8>>>,
storage_root: B::Hash,
validation_params: ValidationParams,
struct WitnessExt<'a, B: BlockT> {
inner: Ext<'a, B>,
params: &'a ValidationParams,
}
impl<B: BlockT> WitnessStorage<B> {
/// Initialize from the given witness data and storage root.
///
/// Returns an error if given storage root was not found in the witness data.
fn new(
storage_proof: StorageProof,
storage_root: B::Hash,
validation_params: ValidationParams,
) -> Result<Self, &'static str> {
let db = storage_proof.into_memory_db();
if !HashDB::contains(&db, &storage_root, EMPTY_PREFIX) {
return Err("Witness data does not contain given storage root.");
}
Ok(Self {
witness_data: db,
overlay: Default::default(),
storage_root,
validation_params,
})
}
/// Find the next storage key after the given `key` in the trie.
fn trie_next_key(&self, key: &[u8]) -> Option<Vec<u8>> {
let trie = TrieDB::<Layout<HashFor<B>>>::new(&self.witness_data, &self.storage_root)
.expect("Creates next storage key `TrieDB`");
let mut iter = trie.iter().expect("Creates trie iterator");
// The key just after the one given in input, basically `key++0`.
// Note: We are sure this is the next key if:
// * size of key has no limit (i.e. we can always add 0 to the path),
// * and no keys can be inserted between `key` and `key++0` (this is ensured by sp-io).
let mut potential_next_key = Vec::with_capacity(key.len() + 1);
potential_next_key.extend_from_slice(key);
potential_next_key.push(0);
iter.seek(&potential_next_key).expect("Seek trie iterator");
let next_element = iter.next();
if let Some(next_element) = next_element {
let (next_key, _) = next_element.expect("Extracts next key");
Some(next_key)
} else {
None
}
}
/// Find the next storage key after the given `key` in the overlay.
fn overlay_next_key(&self, key: &[u8]) -> Option<(&[u8], Option<&[u8]>)> {
let range = (Bound::Excluded(key), Bound::Unbounded);
self.overlay
.range::<[u8], _>(range)
.next()
.map(|(k, v)| (&k[..], v.as_deref()))
}
impl<'a, B: BlockT> WitnessExt<'a, B> {
/// Checks that the encoded `ValidationData` in `data` is correct.
///
/// Should be removed with: https://github.com/paritytech/cumulus/issues/217
/// When removed `WitnessExt` could also be removed.
fn check_validation_data(&self, mut data: &[u8]) {
let validation_data = ValidationData::decode(&mut data).expect("Invalid `ValidationData`");
assert_eq!(
self.validation_params.parent_head,
self.params.parent_head,
validation_data.persisted.parent_head
);
assert_eq!(
self.validation_params.relay_chain_height,
self.params.relay_chain_height,
validation_data.persisted.block_number
);
assert_eq!(
self.validation_params.hrmp_mqc_heads,
self.params.hrmp_mqc_heads,
validation_data.persisted.hrmp_mqc_heads
);
}
}
impl<B: BlockT> Storage for WitnessStorage<B> {
fn modified(&self, key: &[u8]) -> Option<Vec<u8>> {
self.overlay.get(key).cloned().unwrap_or(None)
impl<'a, B: BlockT> Externalities for WitnessExt<'a, B> {
fn storage(&self, key: &[u8]) -> Option<StorageValue> {
self.inner.storage(key)
}
fn get(&self, key: &[u8]) -> Option<Vec<u8>> {
self.overlay.get(key).cloned().unwrap_or_else(|| {
read_trie_value::<Layout<HashFor<B>>, _>(&self.witness_data, &self.storage_root, key)
.expect("Reading key from trie.")
})
fn set_offchain_storage(&mut self, key: &[u8], value: Option<&[u8]>) {
self.inner.set_offchain_storage(key, value)
}
fn insert(&mut self, key: &[u8], value: &[u8]) {
if key == VALIDATION_DATA {
self.check_validation_data(value);
fn storage_hash(&self, key: &[u8]) -> Option<Vec<u8>> {
self.inner.storage_hash(key)
}
fn child_storage(
&self,
child_info: &ChildInfo,
key: &[u8],
) -> Option<StorageValue> {
self.inner.child_storage(child_info, key)
}
fn child_storage_hash(
&self,
child_info: &ChildInfo,
key: &[u8],
) -> Option<Vec<u8>> {
self.inner.child_storage_hash(child_info, key)
}
fn exists_storage(&self, key: &[u8]) -> bool {
self.inner.exists_storage(key)
}
fn exists_child_storage(
&self,
child_info: &ChildInfo,
key: &[u8],
) -> bool {
self.inner.exists_child_storage(child_info, key)
}
fn next_storage_key(&self, key: &[u8]) -> Option<StorageKey> {
self.inner.next_storage_key(key)
}
fn next_child_storage_key(
&self,
child_info: &ChildInfo,
key: &[u8],
) -> Option<StorageKey> {
self.inner.next_child_storage_key(child_info, key)
}
fn place_storage(&mut self, key: StorageKey, value: Option<StorageValue>) {
if let Some(value) = value.as_ref() {
if key == VALIDATION_DATA {
self.check_validation_data(value);
}
}
self.overlay.insert(key.to_vec(), Some(value.to_vec()));
self.inner.place_storage(key, value)
}
fn remove(&mut self, key: &[u8]) {
self.overlay.insert(key.to_vec(), None);
fn place_child_storage(
&mut self,
child_info: &ChildInfo,
key: StorageKey,
value: Option<StorageValue>,
) {
self.inner.place_child_storage(child_info, key, value)
}
fn storage_root(&mut self) -> Vec<u8> {
let root = delta_trie_root::<Layout<HashFor<B>>, _, _, _, _, _>(
&mut self.witness_data,
self.storage_root.clone(),
self.overlay
.iter()
.map(|(k, v)| (k.as_ref(), v.as_ref().map(|v| v.as_ref()))),
)
.expect("Calculates storage root");
root.encode()
fn kill_child_storage(
&mut self,
child_info: &ChildInfo,
) {
self.inner.kill_child_storage(child_info)
}
fn clear_prefix(&mut self, prefix: &[u8]) {
self.overlay.iter_mut().for_each(|(k, v)| {
if k.starts_with(prefix) {
*v = None;
}
});
let trie = match TrieDB::<Layout<HashFor<B>>>::new(&self.witness_data, &self.storage_root) {
Ok(r) => r,
Err(_) => panic!(),
};
for x in TrieDBIterator::new_prefixed(&trie, prefix).expect("Creates trie iterator") {
if let Ok((key, _)) = x {
self.overlay.insert(key, None);
}
}
self.inner.clear_prefix(prefix)
}
fn storage_append(&mut self, key: &[u8], value: Vec<u8>) {
let value_vec = sp_std::vec![EncodeOpaqueValue(value)];
let overlay = &mut self.overlay;
let witness_data = &self.witness_data;
let storage_root = &self.storage_root;
let current_value = overlay.entry(key.to_vec()).or_insert_with(|| {
read_trie_value::<Layout<HashFor<B>>, _>(witness_data, storage_root, key)
.ok()
.flatten()
});
let item = current_value.take().unwrap_or_default();
*current_value = Some(
match Vec::<EncodeOpaqueValue>::append_or_new(item, &value_vec) {
Ok(item) => item,
Err(_) => value_vec.encode(),
},
);
fn clear_child_prefix(
&mut self,
child_info: &ChildInfo,
prefix: &[u8],
) {
self.inner.clear_child_prefix(child_info, prefix)
}
fn next_key(&self, key: &[u8]) -> Option<Vec<u8>> {
let next_trie_key = self.trie_next_key(key);
let next_overlay_key = self.overlay_next_key(key);
fn storage_append(
&mut self,
key: Vec<u8>,
value: Vec<u8>,
) {
self.inner.storage_append(key, value)
}
match (next_trie_key, next_overlay_key) {
(Some(backend_key), Some(overlay_key)) if &backend_key[..] < overlay_key.0 => {
Some(backend_key)
}
(backend_key, None) => backend_key,
(_, Some(overlay_key)) => {
if overlay_key.1.is_some() {
Some(overlay_key.0.to_vec())
} else {
self.next_key(&overlay_key.0)
}
}
}
fn chain_id(&self) -> u64 {
42
}
fn storage_root(&mut self) -> Vec<u8> {
self.inner.storage_root()
}
fn child_storage_root(
&mut self,
child_info: &ChildInfo,
) -> Vec<u8> {
self.inner.child_storage_root(child_info)
}
fn storage_changes_root(&mut self, parent_hash: &[u8]) -> Result<Option<Vec<u8>>, ()> {
self.inner.storage_changes_root(parent_hash)
}
fn storage_start_transaction(&mut self) {
self.inner.storage_start_transaction()
}
fn storage_rollback_transaction(&mut self) -> Result<(), ()> {
self.inner.storage_rollback_transaction()
}
fn storage_commit_transaction(&mut self) -> Result<(), ()> {
self.inner.storage_commit_transaction()
}
fn wipe(&mut self) {
self.inner.wipe()
}
fn commit(&mut self) {
self.inner.commit()
}
fn read_write_count(&self) -> (u32, u32, u32, u32) {
self.inner.read_write_count()
}
fn reset_read_write_count(&mut self) {
self.inner.reset_read_write_count()
}
fn get_whitelist(&self) -> Vec<TrackedStorageKey> {
self.inner.get_whitelist()
}
fn set_whitelist(&mut self, new: Vec<TrackedStorageKey>) {
self.inner.set_whitelist(new)
}
}
impl<'a, B: BlockT> ExtensionStore for WitnessExt<'a, B> {
fn extension_by_type_id(&mut self, type_id: TypeId) -> Option<&mut dyn Any> {
self.inner.extension_by_type_id(type_id)
}
fn register_extension_with_type_id(
&mut self,
type_id: TypeId,
extension: Box<dyn Extension>,
) -> Result<(), Error> {
self.inner.register_extension_with_type_id(type_id, extension)
}
fn deregister_extension_by_type_id(
&mut self,
type_id: TypeId,
) -> Result<(), Error> {
self.inner.deregister_extension_by_type_id(type_id)
}
}
fn host_storage_read(key: &[u8], value_out: &mut [u8], value_offset: u32) -> Option<u32> {
match with_storage(|storage| storage.get(key)) {
match with_externalities(|ext| ext.storage(key)) {
Some(value) => {
let value_offset = value_offset as usize;
let data = &value[value_offset.min(value.len())..];
@@ -378,38 +394,130 @@ fn host_storage_read(key: &[u8], value_out: &mut [u8], value_offset: u32) -> Opt
}
fn host_storage_set(key: &[u8], value: &[u8]) {
with_storage(|storage| storage.insert(key, value));
with_externalities(|ext| ext.place_storage(key.to_vec(), Some(value.to_vec())))
}
fn host_storage_get(key: &[u8]) -> Option<Vec<u8>> {
with_storage(|storage| storage.get(key).clone())
with_externalities(|ext| ext.storage(key).clone())
}
fn host_storage_exists(key: &[u8]) -> bool {
with_storage(|storage| storage.get(key).is_some())
with_externalities(|ext| ext.exists_storage(key))
}
fn host_storage_clear(key: &[u8]) {
with_storage(|storage| storage.remove(key));
with_externalities(|ext| ext.place_storage(key.to_vec(), None))
}
fn host_storage_root() -> Vec<u8> {
with_storage(|storage| storage.storage_root())
with_externalities(|ext| ext.storage_root())
}
fn host_storage_clear_prefix(prefix: &[u8]) {
with_storage(|storage| storage.clear_prefix(prefix))
with_externalities(|ext| ext.clear_prefix(prefix))
}
fn host_storage_changes_root(_: &[u8]) -> Option<Vec<u8>> {
// TODO implement it properly
None
fn host_storage_changes_root(parent_hash: &[u8]) -> Option<Vec<u8>> {
with_externalities(|ext| ext.storage_changes_root(parent_hash).ok().flatten())
}
fn host_storage_append(key: &[u8], value: Vec<u8>) {
with_storage(|storage| storage.storage_append(key, value));
with_externalities(|ext| ext.storage_append(key.to_vec(), value))
}
fn host_storage_next_key(key: &[u8]) -> Option<Vec<u8>> {
with_storage(|storage| storage.next_key(key))
with_externalities(|ext| ext.next_storage_key(key))
}
fn host_storage_start_transaction() {
with_externalities(|ext| ext.storage_start_transaction())
}
fn host_storage_rollback_transaction() {
with_externalities(|ext| ext.storage_rollback_transaction().ok())
.expect("No open transaction that can be rolled back.");
}
fn host_storage_commit_transaction() {
with_externalities(|ext| ext.storage_commit_transaction().ok())
.expect("No open transaction that can be committed.");
}
fn host_default_child_storage_get(storage_key: &[u8], key: &[u8]) -> Option<Vec<u8>> {
let child_info = ChildInfo::new_default(storage_key);
with_externalities(|ext| ext.child_storage(&child_info, key))
}
fn host_default_child_storage_read(
storage_key: &[u8],
key: &[u8],
value_out: &mut [u8],
value_offset: u32,
) -> Option<u32> {
let child_info = ChildInfo::new_default(storage_key);
match with_externalities(|ext| ext.child_storage(&child_info, key)) {
Some(value) => {
let value_offset = value_offset as usize;
let data = &value[value_offset.min(value.len())..];
let written = sp_std::cmp::min(data.len(), value_out.len());
value_out[..written].copy_from_slice(&data[..written]);
Some(value.len() as u32)
}
None => None,
}
}
fn host_default_child_storage_set(
storage_key: &[u8],
key: &[u8],
value: &[u8],
) {
let child_info = ChildInfo::new_default(storage_key);
with_externalities(|ext| ext.place_child_storage(&child_info, key.to_vec(), Some(value.to_vec())))
}
fn host_default_child_storage_clear(
storage_key: &[u8],
key: &[u8],
) {
let child_info = ChildInfo::new_default(storage_key);
with_externalities(|ext| ext.place_child_storage(&child_info, key.to_vec(), None))
}
fn host_default_child_storage_storage_kill(
storage_key: &[u8],
) {
let child_info = ChildInfo::new_default(storage_key);
with_externalities(|ext| ext.kill_child_storage(&child_info))
}
fn host_default_child_storage_exists(
storage_key: &[u8],
key: &[u8],
) -> bool {
let child_info = ChildInfo::new_default(storage_key);
with_externalities(|ext| ext.exists_child_storage(&child_info, key))
}
fn host_default_child_storage_clear_prefix(
storage_key: &[u8],
prefix: &[u8],
) {
let child_info = ChildInfo::new_default(storage_key);
with_externalities(|ext| ext.clear_child_prefix(&child_info, prefix))
}
fn host_default_child_storage_root(
storage_key: &[u8],
) -> Vec<u8> {
let child_info = ChildInfo::new_default(storage_key);
with_externalities(|ext| ext.child_storage_root(&child_info))
}
fn host_default_child_storage_next_key(
storage_key: &[u8],
key: &[u8],
) -> Option<Vec<u8>> {
let child_info = ChildInfo::new_default(storage_key);
with_externalities(|ext| ext.next_child_storage_key(&child_info, key))
}