LocalCallExecutor and RemoteCallRequest generic over Hasher and NodeCodec (#573)

* LocalCallExecutor and RemoteCallRequest generic over Hasher and NodeCodec

* Fix client/db

* Use new triehash

* Use new triehash
Don't use ethereum-types from rlp

* New triehash

* Use new triehash

* fixes and rlp without ethereum-types

* Lockfile

* lockfile

* Rename enumerated_trie_root to keccak_rlp_enumerated_trie_root

* Rename ordered_trie_root -> keccak_rlp_ordered_trie_root
Rename trie_root -> keccak_rlp_trie_root

* Fix panic message

* Kick the ball one step up the ladder

* Normalize function signatures between wasm and native

* Sort out compilation and test errors

* Cleanup

* Constrain the wasm-version using an ExternTrieCrypto trait that is implemented only for KeccakHasher

* hashdb is not optional

* lockfile

* Runtimes

* Missing bounds

* wasm binaries

* lockfiles

* binaries

* LocalCallExecutor uses concrete hasher/codec

* Use binaries from master

* Externalities is concrete
RuntimeInfo is not generic

* whitespace grumble

* lockfile

* lockfile and binaries
This commit is contained in:
David
2018-09-09 14:56:50 +02:00
committed by Gav Wood
parent ea7aeb8409
commit 393c3b5af7
23 changed files with 115 additions and 56 deletions
+2
View File
@@ -2782,6 +2782,8 @@ version = "0.1.0"
dependencies = [
"ed25519 0.1.0",
"environmental 0.1.0",
"hashdb 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rlp 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc_version 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"substrate-codec 0.1.0",
"substrate-primitives 0.1.0",
+2
View File
@@ -701,6 +701,8 @@ version = "0.1.0"
dependencies = [
"ed25519 0.1.0",
"environmental 0.1.0",
"hashdb 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rlp 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc_version 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"substrate-codec 0.1.0",
"substrate-primitives 0.1.0",
@@ -23,10 +23,10 @@ use state_machine::{self, OverlayedChanges, Ext,
use runtime_io::Externalities;
use executor::{RuntimeVersion, RuntimeInfo};
use patricia_trie::NodeCodec;
use primitives::{KeccakHasher, RlpCodec};
use hashdb::Hasher;
use rlp::Encodable;
use codec::Decode;
use primitives::{KeccakHasher, RlpCodec};
use backend;
use error;
@@ -28,7 +28,6 @@ use primitives::H256;
use patricia_trie::NodeCodec;
use hashdb::Hasher;
use rlp::Encodable;
use primitives::{KeccakHasher, RlpCodec};
use blockchain::Backend as ChainBackend;
use call_executor::{CallExecutor, CallResult};
@@ -37,26 +36,32 @@ use light::fetcher::{Fetcher, RemoteCallRequest};
use executor::RuntimeVersion;
use codec::Decode;
use heapsize::HeapSizeOf;
use std::marker::PhantomData;
/// Call executor that executes methods on remote node, querying execution proof
/// and checking proof by re-executing locally.
pub struct RemoteCallExecutor<B, F> {
pub struct RemoteCallExecutor<B, F, H, C> {
blockchain: Arc<B>,
fetcher: Arc<F>,
_hasher: PhantomData<H>,
_codec: PhantomData<C>,
}
impl<B, F> RemoteCallExecutor<B, F> {
impl<B, F, H, C> RemoteCallExecutor<B, F, H, C> {
/// Creates new instance of remote call executor.
pub fn new(blockchain: Arc<B>, fetcher: Arc<F>) -> Self {
RemoteCallExecutor { blockchain, fetcher }
RemoteCallExecutor { blockchain, fetcher, _hasher: PhantomData, _codec: PhantomData }
}
}
impl<B, F, Block> CallExecutor<Block, KeccakHasher, RlpCodec> for RemoteCallExecutor<B, F>
where
Block: BlockT,
B: ChainBackend<Block>,
F: Fetcher<Block>,
impl<B, F, Block, H, C> CallExecutor<Block, H, C> for RemoteCallExecutor<B, F, H, C>
where
Block: BlockT,
B: ChainBackend<Block>,
F: Fetcher<Block>,
H: Hasher,
H::Out: Ord + Encodable,
C: NodeCodec<H>
{
type Error = ClientError;
@@ -84,7 +89,7 @@ impl<B, F, Block> CallExecutor<Block, KeccakHasher, RlpCodec> for RemoteCallExec
}
fn call_at_state<
S: StateBackend<KeccakHasher, RlpCodec>,
S: StateBackend<H, C>,
FF: FnOnce(Result<Vec<u8>, Self::Error>, Result<Vec<u8>, Self::Error>) -> Result<Vec<u8>, Self::Error>
>(&self,
_state: &S,
@@ -96,7 +101,7 @@ impl<B, F, Block> CallExecutor<Block, KeccakHasher, RlpCodec> for RemoteCallExec
Err(ClientErrorKind::NotAvailableOnLightClient.into())
}
fn prove_at_state<S: StateBackend<KeccakHasher, RlpCodec>>(
fn prove_at_state<S: StateBackend<H, C>>(
&self,
_state: S,
_changes: &mut OverlayedChanges,
+2 -1
View File
@@ -23,6 +23,7 @@ pub mod fetcher;
use std::sync::Arc;
use primitives::{KeccakHasher, RlpCodec};
use runtime_primitives::BuildStorage;
use runtime_primitives::traits::Block as BlockT;
use state_machine::{CodeExecutor, ExecutionStrategy};
@@ -52,7 +53,7 @@ pub fn new_light<B, S, F, GS>(
backend: Arc<Backend<S, F>>,
fetcher: Arc<F>,
genesis_storage: GS,
) -> ClientResult<Client<Backend<S, F>, RemoteCallExecutor<Blockchain<S, F>, F>, B>>
) -> ClientResult<Client<Backend<S, F>, RemoteCallExecutor<Blockchain<S, F>, F, KeccakHasher, RlpCodec>, B>>
where
B: BlockT,
S: BlockchainStorage<B>,
@@ -293,7 +293,7 @@ impl_function_executor!(this: FunctionExecutor<'e, E>,
this.memory.set(result, r.as_ref()).map_err(|_| UserError("Invalid attempt to set memory in ext_storage_root"))?;
Ok(())
},
ext_enumerated_trie_root(values_data: *const u8, lens_data: *const u32, lens_len: u32, result: *mut u8) => {
ext_keccak_enumerated_trie_root(values_data: *const u8, lens_data: *const u32, lens_len: u32, result: *mut u8) => {
let values = (0..lens_len)
.map(|i| this.memory.read_primitive(lens_data + i * 4))
.collect::<::std::result::Result<Vec<u32>, UserError>>()?
@@ -301,11 +301,11 @@ impl_function_executor!(this: FunctionExecutor<'e, E>,
.scan(0u32, |acc, v| { let o = *acc; *acc += v; Some((o, v)) })
.map(|(offset, len)|
this.memory.get(values_data + offset, len as usize)
.map_err(|_| UserError("Invalid attempt to get memory in ext_enumerated_trie_root"))
.map_err(|_| UserError("Invalid attempt to get memory in ext_keccak_enumerated_trie_root"))
)
.collect::<::std::result::Result<Vec<_>, UserError>>()?;
let r = ordered_trie_root::<KeccakHasher, _, _>(values.into_iter());
this.memory.set(result, &r[..]).map_err(|_| UserError("Invalid attempt to set memory in ext_enumerated_trie_root"))?;
this.memory.set(result, &r[..]).map_err(|_| UserError("Invalid attempt to set memory in ext_keccak_enumerated_trie_root"))?;
Ok(())
},
ext_chain_id() -> u64 => {
+2
View File
@@ -71,6 +71,7 @@ dependencies = [
name = "runtime-test"
version = "0.1.0"
dependencies = [
"substrate-primitives 0.1.0",
"substrate-runtime-io 0.1.0",
"substrate-runtime-sandbox 0.1.0",
]
@@ -143,6 +144,7 @@ dependencies = [
name = "substrate-runtime-io"
version = "0.1.0"
dependencies = [
"hashdb 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc_version 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"substrate-codec 0.1.0",
"substrate-primitives 0.1.0",
@@ -9,6 +9,7 @@ crate-type = ["cdylib"]
[dependencies]
substrate-runtime-io = { path = "../../runtime-io", version = "0.1", default_features = false }
substrate-runtime-sandbox = { path = "../../runtime-sandbox", version = "0.1", default_features = false }
substrate-primitives = { path = "../../primitives", default_features = false }
[profile.release]
panic = "abort"
+2 -1
View File
@@ -9,6 +9,7 @@ use alloc::vec::Vec;
#[macro_use]
extern crate substrate_runtime_io as runtime_io;
extern crate substrate_runtime_sandbox as sandbox;
extern crate substrate_primitives;
use runtime_io::{
set_storage, storage, clear_prefix, print, blake2_256,
@@ -55,7 +56,7 @@ impl_stubs!(
[ed25519_verify(&sig, &msg[..], &pubkey) as u8].to_vec()
},
test_enumerated_trie_root NO_DECODE => |_| {
enumerated_trie_root(&[&b"zero"[..], &b"one"[..], &b"two"[..]]).to_vec()
enumerated_trie_root::<substrate_primitives::KeccakHasher>(&[&b"zero"[..], &b"one"[..], &b"two"[..]]).to_vec()
},
test_sandbox NO_DECODE => |code: &[u8]| {
let ok = execute_sandboxed(code, &[]).is_ok();
@@ -15,6 +15,8 @@ substrate-primitives = { path = "../primitives", default_features = false }
substrate-codec = { path = "../codec", default_features = false }
triehash = { version = "0.2", optional = true }
ed25519 = { path = "../ed25519", optional = true }
hashdb = { version = "0.2", default_features = false }
rlp = { version = "0.2", optional = true, default_features = false }
[features]
default = ["std"]
@@ -26,6 +28,7 @@ std = [
"substrate-codec/std",
"substrate-runtime-std/std",
"ed25519",
"rlp"
]
nightly = []
strict = []
+32 -17
View File
@@ -23,6 +23,8 @@ extern crate substrate_primitives as primitives;
extern crate substrate_state_machine;
extern crate triehash;
extern crate ed25519;
extern crate hashdb;
extern crate rlp;
#[doc(hidden)]
pub extern crate substrate_codec as codec;
@@ -35,6 +37,8 @@ pub use primitives::KeccakHasher;
pub use substrate_state_machine::{Externalities, TestExternalities};
use primitives::hexdisplay::HexDisplay;
use primitives::H256;
use hashdb::Hasher;
use rlp::Encodable;
// TODO: use the real error, not NoError.
@@ -43,12 +47,13 @@ environmental!(ext: trait Externalities<KeccakHasher>);
/// Get `key` from storage and return a `Vec`, empty if there's a problem.
pub fn storage(key: &[u8]) -> Option<Vec<u8>> {
ext::with(|ext| ext.storage(key).map(|s| s.to_vec()))
.expect("read_storage cannot be called outside of an Externalities-provided environment.")
.expect("storage cannot be called outside of an Externalities-provided environment.")
}
/// Get `key` from storage, placing the value into `value_out` (as much as possible) and return
/// the number of bytes that the key in storage was beyond the offset or None if the storage entry
/// doesn't exist at all.
/// Get `key` from storage, placing the value into `value_out` (as much of it as possible) and return
/// the number of bytes that the entry in storage had beyond the offset or None if the storage entry
/// doesn't exist at all. Note that if the buffer is smaller than the storage entry length, the returned
/// number of bytes is not equal to the number of bytes written to the `value_out`.
pub fn read_storage(key: &[u8], value_out: &mut [u8], value_offset: usize) -> Option<usize> {
ext::with(|ext| ext.storage(key).map(|value| {
let value = &value[value_offset..];
@@ -58,14 +63,14 @@ pub fn read_storage(key: &[u8], value_out: &mut [u8], value_offset: usize) -> Op
})).expect("read_storage cannot be called outside of an Externalities-provided environment.")
}
/// Set the storage of some particular key to Some value.
/// Set the storage of a key to some value.
pub fn set_storage(key: &[u8], value: &[u8]) {
ext::with(|ext|
ext.set_storage(key.to_vec(), value.to_vec())
);
}
/// Clear the storage of some particular key.
/// Clear the storage of a key.
pub fn clear_storage(key: &[u8]) {
ext::with(|ext|
ext.clear_storage(key)
@@ -79,7 +84,7 @@ pub fn exists_storage(key: &[u8]) -> bool {
).unwrap_or(false)
}
/// Clear the storage entries key of which starts with the given prefix.
/// Clear the storage entries with a key that starts with the given prefix.
pub fn clear_prefix(prefix: &[u8]) {
ext::with(|ext|
ext.clear_prefix(prefix)
@@ -93,7 +98,7 @@ pub fn chain_id() -> u64 {
).unwrap_or(0)
}
/// "Commit" all existing operations and get the resultant storage root.
/// "Commit" all existing operations and compute the resultant storage root.
pub fn storage_root() -> H256 {
ext::with(|ext|
ext.storage_root()
@@ -101,25 +106,35 @@ pub fn storage_root() -> H256 {
}
/// A trie root formed from the enumerated items.
pub fn enumerated_trie_root(serialised_values: &[&[u8]]) -> [u8; 32] {
triehash::ordered_trie_root::<KeccakHasher, _, _>(serialised_values.iter().map(|s| s.to_vec())).0
pub fn enumerated_trie_root<H>(serialised_values: &[&[u8]]) -> H::Out
where
H: Hasher,
H::Out: Encodable + Ord,
{
triehash::ordered_trie_root::<H, _, _>(serialised_values.iter().map(|s| s.to_vec()))
}
/// A trie root formed from the iterated items.
pub fn trie_root<
pub fn trie_root<H, I, A, B>(input: I) -> H::Out
where
I: IntoIterator<Item = (A, B)>,
A: AsRef<[u8]> + Ord,
B: AsRef<[u8]>,
>(input: I) -> [u8; 32] {
triehash::trie_root::<KeccakHasher, _, _, _>(input).0
H: Hasher,
<H as Hasher>::Out: Encodable + Ord,
{
triehash::trie_root::<H, _, _, _>(input)
}
/// A trie root formed from the enumerated items.
pub fn ordered_trie_root<
pub fn ordered_trie_root<H, I, A>(input: I) -> H::Out
where
I: IntoIterator<Item = A>,
A: AsRef<[u8]>
>(input: I) -> [u8; 32] {
triehash::ordered_trie_root::<KeccakHasher, _, _>(input).0
A: AsRef<[u8]>,
H: Hasher,
<H as Hasher>::Out: Encodable + Ord,
{
triehash::ordered_trie_root::<H, _, _>(input)
}
/// Verify a ed25519 signature.
+31 -13
View File
@@ -16,6 +16,7 @@
extern crate substrate_primitives as primitives;
extern crate hashdb;
#[doc(hidden)]
pub extern crate substrate_runtime_std as rstd;
@@ -25,6 +26,8 @@ pub extern crate substrate_codec as codec;
use core::intrinsics;
use rstd::vec::Vec;
use hashdb::Hasher;
use primitives::KeccakHasher;
pub use rstd::{mem, slice};
#[panic_handler]
@@ -61,7 +64,7 @@ extern "C" {
fn ext_get_allocated_storage(key_data: *const u8, key_len: u32, written_out: *mut u32) -> *mut u8;
fn ext_get_storage_into(key_data: *const u8, key_len: u32, value_data: *mut u8, value_len: u32, value_offset: u32) -> u32;
fn ext_storage_root(result: *mut u8);
fn ext_enumerated_trie_root(values_data: *const u8, lens_data: *const u32, lens_len: u32, result: *mut u8);
fn ext_keccak_enumerated_trie_root(values_data: *const u8, lens_data: *const u32, lens_len: u32, result: *mut u8);
fn ext_chain_id() -> u64;
fn ext_blake2_256(data: *const u8, len: u32, out: *mut u8);
fn ext_twox_128(data: *const u8, len: u32, out: *mut u8);
@@ -69,6 +72,29 @@ extern "C" {
fn ext_ed25519_verify(msg_data: *const u8, msg_len: u32, sig_data: *const u8, pubkey_data: *const u8) -> u32;
}
/// Ensures we use the right crypto when calling into native
pub trait ExternTrieCrypto {
fn enumerated_trie_root(values: &[&[u8]]) -> [u8; 32];
}
// Ensures we use a Keccak-flavoured Hasher when calling into native
impl ExternTrieCrypto for KeccakHasher {
fn enumerated_trie_root(values: &[&[u8]]) -> [u8; 32] {
let lengths = values.iter().map(|v| (v.len() as u32).to_le()).collect::<Vec<_>>();
let values = values.iter().fold(Vec::new(), |mut acc, sl| { acc.extend_from_slice(sl); acc });
let mut result: [u8; 32] = Default::default();
unsafe {
ext_keccak_enumerated_trie_root(
values.as_ptr(),
lengths.as_ptr(),
lengths.len() as u32,
result.as_mut_ptr()
);
}
result
}
}
/// Get `key` from storage and return a `Vec`, empty if there's a problem.
pub fn storage(key: &[u8]) -> Option<Vec<u8>> {
let mut length: u32 = 0;
@@ -145,22 +171,13 @@ pub fn storage_root() -> [u8; 32] {
}
/// A trie root calculated from enumerated values.
pub fn enumerated_trie_root(values: &[&[u8]]) -> [u8; 32] {
let lens = values.iter().map(|v| (v.len() as u32).to_le()).collect::<Vec<_>>();
let values = values.iter().fold(Vec::new(), |mut acc, sl| { acc.extend_from_slice(sl); acc });
let mut result: [u8; 32] = Default::default();
unsafe {
ext_enumerated_trie_root(
values.as_ptr(),
lens.as_ptr(), lens.len() as u32,
result.as_mut_ptr()
);
}
result
pub fn enumerated_trie_root<H: Hasher + ExternTrieCrypto>(values: &[&[u8]]) -> [u8; 32] {
H::enumerated_trie_root(values)
}
/// A trie root formed from the iterated items.
pub fn trie_root<
H: Hasher + ExternTrieCrypto,
I: IntoIterator<Item = (A, B)>,
A: AsRef<[u8]> + Ord,
B: AsRef<[u8]>,
@@ -172,6 +189,7 @@ pub fn trie_root<
/// A trie root formed from the enumerated items.
pub fn ordered_trie_root<
H: Hasher + ExternTrieCrypto,
I: IntoIterator<Item = A>,
A: AsRef<[u8]>
>(_input: I) -> [u8; 32] {
@@ -22,6 +22,7 @@ use runtime_io;
#[cfg(feature = "std")] use std::fmt::{Debug, Display};
#[cfg(feature = "std")] use serde::{Serialize, de::DeserializeOwned};
use substrate_primitives;
use substrate_primitives::KeccakHasher;
use codec::{Codec, Encode};
pub use integer_sqrt::IntegerSquareRoot;
pub use num_traits::{Zero, One, Bounded};
@@ -230,20 +231,20 @@ impl Hash for BlakeTwo256 {
runtime_io::blake2_256(s).into()
}
fn enumerated_trie_root(items: &[&[u8]]) -> Self::Output {
runtime_io::enumerated_trie_root(items).into()
runtime_io::enumerated_trie_root::<KeccakHasher>(items).into()
}
fn trie_root<
I: IntoIterator<Item = (A, B)>,
A: AsRef<[u8]> + Ord,
B: AsRef<[u8]>
>(input: I) -> Self::Output {
runtime_io::trie_root(input).into()
runtime_io::trie_root::<KeccakHasher, _, _, _>(input).into()
}
fn ordered_trie_root<
I: IntoIterator<Item = A>,
A: AsRef<[u8]>
>(input: I) -> Self::Output {
runtime_io::ordered_trie_root(input).into()
runtime_io::ordered_trie_root::<KeccakHasher, _, _>(input).into()
}
fn storage_root() -> Self::Output {
runtime_io::storage_root().into()
@@ -49,7 +49,7 @@ pub type FullBackend<F> = client_db::Backend<<F as ServiceFactory>::Block>;
/// Full client executor type for a factory.
pub type FullExecutor<F> = client::LocalCallExecutor<
client_db::Backend<<F as ServiceFactory>::Block>,
CodeExecutor<F>
CodeExecutor<F>,
>;
/// Light client backend type for a factory.
@@ -64,7 +64,9 @@ pub type LightExecutor<F> = client::light::call_executor::RemoteCallExecutor<
client_db::light::LightStorage<<F as ServiceFactory>::Block>,
network::OnDemand<<F as ServiceFactory>::Block, NetworkService<F>>
>,
network::OnDemand<<F as ServiceFactory>::Block, NetworkService<F>>
network::OnDemand<<F as ServiceFactory>::Block, NetworkService<F>>,
KeccakHasher,
RlpCodec,
>;
/// Full client type for a factory.
@@ -144,7 +146,7 @@ pub trait ServiceFactory: 'static {
/// Build network protocol.
fn build_network_protocol(config: &FactoryFullConfiguration<Self>)
-> Result<Self::NetworkProtocol, error::Error>;
}
}
/// A collection of types and function to generalise over full / light client type.
pub trait Components: 'static {
+4 -1
View File
@@ -53,7 +53,10 @@ pub use local_executor::LocalExecutor;
pub type Backend = client::in_mem::Backend<runtime::Block, KeccakHasher, RlpCodec>;
/// Test client executor.
pub type Executor = client::LocalCallExecutor<Backend, executor::NativeExecutor<LocalExecutor>>;
pub type Executor = client::LocalCallExecutor<
Backend,
executor::NativeExecutor<LocalExecutor>,
>;
/// Creates new client instance used for tests.
pub fn new() -> client::Client<Backend, Executor, runtime::Block> {
@@ -24,6 +24,7 @@ use runtime_primitives::traits::{Hash as HashT, BlakeTwo256};
use runtime_primitives::{ApplyError, ApplyOutcome, ApplyResult};
use codec::{KeyedVec, Encode};
use super::{AccountId, BlockNumber, Extrinsic, H256 as Hash, Block, Header};
use primitives::KeccakHasher;
const NONCE_OF: &[u8] = b"nonce:";
const BALANCE_OF: &[u8] = b"balance:";
@@ -68,7 +69,7 @@ pub fn execute_block(block: Block) {
// check transaction trie root represents the transactions.
let txs = block.extrinsics.iter().map(Encode::encode).collect::<Vec<_>>();
let txs = txs.iter().map(Vec::as_slice).collect::<Vec<_>>();
let txs_root = enumerated_trie_root(&txs).into();
let txs_root = enumerated_trie_root::<KeccakHasher>(&txs).into();
info_expect_equal_hash(&txs_root, &header.extrinsics_root);
assert!(txs_root == header.extrinsics_root, "Transaction trie root must be valid.");
@@ -95,7 +96,7 @@ pub fn finalise_block() -> Header {
let extrinsic_index = ExtrinsicIndex::take();
let txs: Vec<_> = (0..extrinsic_index).map(ExtrinsicData::take).collect();
let txs = txs.iter().map(Vec::as_slice).collect::<Vec<_>>();
let extrinsics_root = enumerated_trie_root(&txs).into();
let extrinsics_root = enumerated_trie_root::<KeccakHasher>(&txs).into();
let number = <Number>::take();
let parent_hash = <ParentHash>::take();
+2
View File
@@ -553,6 +553,8 @@ version = "0.1.0"
dependencies = [
"ed25519 0.1.0",
"environmental 0.1.0",
"hashdb 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rlp 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc_version 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"substrate-codec 0.1.0",
"substrate-primitives 0.1.0",