Block builder (substrate) (#73)

* Block builder (substrate)

* Fix wasm build

* Bulid on any block

* Test for block builder.

* Block import tests for client.

* Tidy ups

* clean up block builder instantiation

* Propert block generation for tests

* Fixed rpc tests
This commit is contained in:
Gav Wood
2018-02-15 16:38:19 +01:00
committed by Robert Habermeier
parent 03a51a0db8
commit c56859f331
25 changed files with 422 additions and 136 deletions
+4 -2
View File
@@ -13,8 +13,10 @@ ed25519 = { path = "../ed25519" }
substrate-codec = { path = "../codec" }
substrate-executor = { path = "../executor" }
substrate-primitives = { path = "../primitives" }
substrate-runtime-io = { path = "../runtime-io" }
substrate-runtime-support = { path = "../runtime-support" }
substrate-serializer = { path = "../serializer" }
substrate-state-machine = { path = "../state-machine" }
substrate-test-runtime = { path = "../test-runtime" }
substrate-keyring = { path = "../../substrate/keyring" }
[dev-dependencies]
substrate-test-runtime = { path = "../test-runtime" }
+8 -6
View File
@@ -21,22 +21,24 @@ use error;
use primitives::block;
use blockchain::{self, BlockId};
/// Block insertion transction. Keeps hold if the inserted block state and data.
/// Block insertion operation. Keeps hold if the inserted block state and data.
pub trait BlockImportOperation {
/// Associated state backend type.
type State: state_machine::backend::Backend;
/// Returns pending state.
fn state(&self) -> error::Result<Self::State>;
fn state(&self) -> error::Result<&Self::State>;
/// Append block data to the transaction.
fn import_block(&mut self, header: block::Header, body: Option<block::Body>, is_new_best: bool) -> error::Result<()>;
fn set_block_data(&mut self, header: block::Header, body: Option<block::Body>, is_new_best: bool) -> error::Result<()>;
/// Inject storage data into the database.
fn set_storage<I: Iterator<Item=(Vec<u8>, Vec<u8>)>>(&mut self, iter: I) -> error::Result<()>;
/// Inject storage data into the database.
fn reset_storage<I: Iterator<Item=(Vec<u8>, Vec<u8>)>>(&mut self, iter: I) -> error::Result<()>;
}
/// Client backend. Manages the data layer.
pub trait Backend {
/// Associated block insertion transaction type.
/// Associated block insertion operation type.
type BlockImportOperation: BlockImportOperation;
/// Associated blockchain backend type.
type Blockchain: blockchain::Backend;
@@ -44,9 +46,9 @@ pub trait Backend {
type State: state_machine::backend::Backend;
/// Begin a new block insertion transaction with given parent block id.
fn begin_transaction(&self, block: BlockId) -> error::Result<Self::BlockImportOperation>;
fn begin_operation(&self, block: BlockId) -> error::Result<Self::BlockImportOperation>;
/// Commit block insertion.
fn commit_transaction(&self, transaction: Self::BlockImportOperation) -> error::Result<()>;
fn commit_operation(&self, transaction: Self::BlockImportOperation) -> error::Result<()>;
/// Returns reference to blockchain backend.
fn blockchain(&self) -> &Self::Blockchain;
/// Returns state backend for specified block.
@@ -0,0 +1,90 @@
// Copyright 2017 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Utility struct to build a block.
use std::vec::Vec;
use codec::{Joiner, Slicable};
use state_machine::{self, CodeExecutor};
use primitives::{Header, Block};
use primitives::block::Transaction;
use {backend, error, BlockId, Client};
use triehash::ordered_trie_root;
/// Utility for building new (valid) blocks from a stream of transactions.
pub struct BlockBuilder<B, E> where
B: backend::Backend,
E: CodeExecutor + Clone,
error::Error: From<<<B as backend::Backend>::State as state_machine::backend::Backend>::Error>,
{
header: Header,
transactions: Vec<Transaction>,
executor: E,
state: B::State,
changes: state_machine::OverlayedChanges,
}
impl<B, E> BlockBuilder<B, E> where
B: backend::Backend,
E: CodeExecutor + Clone,
error::Error: From<<<B as backend::Backend>::State as state_machine::backend::Backend>::Error>,
{
/// Create a new instance of builder from the given client, building on the latest block.
pub fn new(client: &Client<B, E>) -> error::Result<Self> {
client.info().and_then(|i| Self::at_block(&BlockId::Hash(i.chain.best_hash), client))
}
/// Create a new instance of builder from the given client using a particular block's ID to
/// build upon.
pub fn at_block(block_id: &BlockId, client: &Client<B, E>) -> error::Result<Self> {
Ok(BlockBuilder {
header: Header {
number: client.block_number_from_id(block_id)?.ok_or(error::ErrorKind::UnknownBlock(*block_id))? + 1,
parent_hash: client.block_hash_from_id(block_id)?.ok_or(error::ErrorKind::UnknownBlock(*block_id))?,
state_root: Default::default(),
transaction_root: Default::default(),
digest: Default::default(),
},
transactions: Default::default(),
executor: client.clone_executor(),
state: client.state_at(block_id)?,
changes: Default::default(),
})
}
/// Push a transaction onto the block's list of transactions. This will ensure the transaction
/// can be validly executed (by executing it); if it is invalid, it'll be returned along with
/// the error. Otherwise, it will return a mutable reference to self (in order to chain).
pub fn push(&mut self, tx: Transaction) -> error::Result<()> {
let output = state_machine::execute(&self.state, &mut self.changes, &self.executor, "execute_transaction",
&vec![].and(&self.header).and(&tx))?;
self.header = Header::decode(&mut &output[..]).expect("Header came straight out of runtime so must be valid");
self.transactions.push(tx);
Ok(())
}
/// Consume the builder to return a valid `Block` containing all pushed transactions.
pub fn bake(mut self) -> error::Result<Block> {
self.header.transaction_root = ordered_trie_root(self.transactions.iter().map(Slicable::encode)).0.into();
let output = state_machine::execute(&self.state, &mut self.changes, &self.executor, "finalise_block",
&self.header.encode())?;
self.header = Header::decode(&mut &output[..]).expect("Header came straight out of runtime so must be valid");
Ok(Block {
header: self.header,
transactions: self.transactions,
})
}
}
+17 -45
View File
@@ -21,13 +21,13 @@ use parking_lot::RwLock;
use state_machine;
use error;
use backend;
use primitives;
use ser;
use runtime_support::Hashable;
use primitives::block::{self, HeaderHash};
use blockchain::{self, BlockId, BlockStatus};
use state_machine::backend::Backend as StateBackend;
fn header_hash(header: &block::Header) -> block::HeaderHash {
primitives::hashing::blake2_256(&ser::encode(header)).into()
header.blake2_256().into()
}
struct PendingBlock {
@@ -41,7 +41,7 @@ struct Block {
body: Option<block::Body>,
}
/// In-memory transaction.
/// In-memory operation.
pub struct BlockImportOperation {
pending_block: Option<PendingBlock>,
pending_state: state_machine::backend::InMemory,
@@ -156,12 +156,12 @@ impl blockchain::Backend for Blockchain {
impl backend::BlockImportOperation for BlockImportOperation {
type State = state_machine::backend::InMemory;
fn state(&self) -> error::Result<Self::State> {
Ok(self.pending_state.clone())
fn state(&self) -> error::Result<&Self::State> {
Ok(&self.pending_state)
}
fn import_block(&mut self, header: block::Header, body: Option<block::Body>, is_new_best: bool) -> error::Result<()> {
assert!(self.pending_block.is_none(), "Only one block per transaction is allowed");
fn set_block_data(&mut self, header: block::Header, body: Option<block::Body>, is_new_best: bool) -> error::Result<()> {
assert!(self.pending_block.is_none(), "Only one block per operation is allowed");
self.pending_block = Some(PendingBlock {
block: Block {
header: header,
@@ -172,6 +172,11 @@ impl backend::BlockImportOperation for BlockImportOperation {
Ok(())
}
fn set_storage<I: Iterator<Item=(Vec<u8>, Vec<u8>)>>(&mut self, changes: I) -> error::Result<()> {
self.pending_state.commit(changes);
Ok(())
}
fn reset_storage<I: Iterator<Item=(Vec<u8>, Vec<u8>)>>(&mut self, iter: I) -> error::Result<()> {
self.pending_state = state_machine::backend::InMemory::from(iter.collect());
Ok(())
@@ -192,39 +197,6 @@ impl Backend {
blockchain: Blockchain::new(),
}
}
/// Generate and import a sequence of blocks. A user supplied function is allowed to modify each block header. Useful for testing.
pub fn generate_blocks<F>(&self, count: usize, edit_header: F) where F: Fn(&mut block::Header) {
use backend::{Backend, BlockImportOperation};
let info = blockchain::Backend::info(&self.blockchain).expect("In-memory backend never fails");
let mut best_num = info.best_number;
let mut best_hash = info.best_hash;
let state_root = blockchain::Backend::header(&self.blockchain, BlockId::Hash(best_hash))
.expect("In-memory backend never fails")
.expect("Best header always exists in the blockchain")
.state_root;
for _ in 0 .. count {
best_num = best_num + 1;
let mut header = block::Header {
parent_hash: best_hash,
number: best_num,
state_root: state_root,
transaction_root: Default::default(),
digest: Default::default(),
};
edit_header(&mut header);
let mut tx = self.begin_transaction(BlockId::Hash(best_hash)).expect("In-memory backend does not fail");
best_hash = header_hash(&header);
tx.import_block(header, None, true).expect("In-memory backend does not fail");
self.commit_transaction(tx).expect("In-memory backend does not fail");
}
}
/// Generate and import a sequence of blocks. Useful for testing.
pub fn push_blocks(&self, count: usize) {
self.generate_blocks(count, |_| {})
}
}
impl backend::Backend for Backend {
@@ -232,7 +204,7 @@ impl backend::Backend for Backend {
type Blockchain = Blockchain;
type State = state_machine::backend::InMemory;
fn begin_transaction(&self, block: BlockId) -> error::Result<Self::BlockImportOperation> {
fn begin_operation(&self, block: BlockId) -> error::Result<Self::BlockImportOperation> {
let state = match block {
BlockId::Hash(h) if h.is_zero() => Self::State::default(),
_ => self.state_at(block)?,
@@ -244,10 +216,10 @@ impl backend::Backend for Backend {
})
}
fn commit_transaction(&self, transaction: Self::BlockImportOperation) -> error::Result<()> {
if let Some(pending_block) = transaction.pending_block {
fn commit_operation(&self, operation: Self::BlockImportOperation) -> error::Result<()> {
if let Some(pending_block) = operation.pending_block {
let hash = header_hash(&pending_block.block.header);
self.states.write().insert(hash, transaction.pending_state);
self.states.write().insert(hash, operation.pending_state);
self.blockchain.insert(hash, pending_block.block.header, pending_block.block.body, pending_block.is_best);
}
Ok(())
+169 -14
View File
@@ -18,13 +18,13 @@
#![warn(missing_docs)]
extern crate substrate_runtime_support as runtime_support;
extern crate substrate_runtime_io as runtime_io;
extern crate substrate_primitives as primitives;
extern crate substrate_state_machine as state_machine;
extern crate substrate_serializer as ser;
extern crate substrate_codec as codec;
#[cfg(test)] #[macro_use] extern crate substrate_executor as executor;
extern crate ed25519;
#[cfg(test)] extern crate substrate_runtime_support as runtime_support;
#[cfg(test)] extern crate substrate_test_runtime as test_runtime;
#[cfg(test)] extern crate substrate_keyring as keyring;
@@ -39,6 +39,7 @@ pub mod blockchain;
pub mod backend;
pub mod in_mem;
pub mod genesis;
pub mod block_builder;
pub use blockchain::Info as ChainInfo;
pub use blockchain::BlockId;
@@ -50,6 +51,8 @@ use codec::{KeyedVec, Slicable};
use blockchain::Backend as BlockchainBackend;
use backend::BlockImportOperation;
use state_machine::backend::Backend as StateBackend;
use state_machine::{Ext, OverlayedChanges};
use runtime_support::Hashable;
/// Polkadot Client
#[derive(Debug)]
@@ -59,6 +62,7 @@ pub struct Client<B, E> where B: backend::Backend {
}
/// Client info
// TODO: split queue info from chain info and amalgamate into single struct.
#[derive(Debug)]
pub struct ClientInfo {
/// Best block hash.
@@ -134,10 +138,10 @@ impl<B, E> Client<B, E> where
if backend.blockchain().header(BlockId::Number(0))?.is_none() {
trace!("Empty database, writing genesis block");
let (genesis_header, genesis_store) = build_genesis();
let mut tx = backend.begin_transaction(BlockId::Hash(block::HeaderHash::default()))?;
tx.reset_storage(genesis_store.into_iter())?;
tx.import_block(genesis_header, None, true)?;
backend.commit_transaction(tx)?;
let mut op = backend.begin_operation(BlockId::Hash(block::HeaderHash::default()))?;
op.reset_storage(genesis_store.into_iter())?;
op.set_block_data(genesis_header, Some(vec![]), true)?;
backend.commit_operation(op)?;
}
Ok(Client {
backend,
@@ -167,6 +171,11 @@ impl<B, E> Client<B, E> where
self.storage(id, &StorageKey(b":code".to_vec())).map(|data| data.0)
}
/// Clone a new instance of Executor.
pub fn clone_executor(&self) -> E where E: Clone {
self.executor.clone()
}
/// Get the current set of authorities from storage.
pub fn authorities_at(&self, id: &BlockId) -> error::Result<Vec<AuthorityId>> {
let state = self.state_at(id)?;
@@ -183,10 +192,8 @@ impl<B, E> Client<B, E> where
/// No changes are made.
pub fn call(&self, id: &BlockId, method: &str, call_data: &[u8]) -> error::Result<CallResult> {
let mut changes = state_machine::OverlayedChanges::default();
let state = self.state_at(id)?;
let return_data = state_machine::execute(
&state,
&self.state_at(id)?,
&mut changes,
&self.executor,
method,
@@ -195,6 +202,33 @@ impl<B, E> Client<B, E> where
Ok(CallResult { return_data, changes })
}
/// Set up the native execution environment to call into a native runtime code.
pub fn using_environment<F: FnOnce() -> T, T>(
&self, f: F
) -> error::Result<T> {
self.using_environment_at(&BlockId::Number(self.info()?.chain.best_number), &mut Default::default(), f)
}
/// Set up the native execution environment to call into a native runtime code.
pub fn using_environment_at<F: FnOnce() -> T, T>(
&self,
id: &BlockId,
overlay: &mut OverlayedChanges,
f: F
) -> error::Result<T> {
Ok(runtime_io::with_externalities(&mut Ext { backend: &self.state_at(id)?, overlay }, f))
}
/// Create a new block, built on the head of the chain.
pub fn new_block(&self) -> error::Result<block_builder::BlockBuilder<B, E>> where E: Clone {
block_builder::BlockBuilder::new(self)
}
/// Create a new block, built on top of `parent`.
pub fn new_block_at(&self, parent: &BlockId) -> error::Result<block_builder::BlockBuilder<B, E>> where E: Clone {
block_builder::BlockBuilder::at_block(parent, &self)
}
/// Queue a block for import.
pub fn import_block(&self, header: block::Header, body: Option<block::Body>) -> error::Result<ImportResult> {
// TODO: import lock
@@ -204,13 +238,22 @@ impl<B, E> Client<B, E> where
blockchain::BlockStatus::Unknown => return Ok(ImportResult::UnknownParent),
}
let mut transaction = self.backend.begin_transaction(BlockId::Hash(header.parent_hash))?;
let mut _state = transaction.state()?;
// TODO: execute block on _state
let mut transaction = self.backend.begin_operation(BlockId::Hash(header.parent_hash))?;
let mut overlay = OverlayedChanges::default();
state_machine::execute(
transaction.state()?,
&mut overlay,
&self.executor,
"execute_block",
&block::Block { header: header.clone(), transactions: body.clone().unwrap_or_default().clone() }.encode()
)?;
let is_new_best = header.number == self.backend.blockchain().info()?.best_number + 1;
transaction.import_block(header, body, is_new_best)?;
self.backend.commit_transaction(transaction)?;
trace!("Imported {}, (#{}), best={}", block::HeaderHash::from(header.blake2_256()), header.number, is_new_best);
transaction.set_block_data(header, body, is_new_best)?;
transaction.set_storage(overlay.drain())?;
self.backend.commit_operation(transaction)?;
Ok(ImportResult::Queued)
}
@@ -238,6 +281,22 @@ impl<B, E> Client<B, E> where
self.backend.blockchain().hash(block_number)
}
/// Convert an arbitrary block ID into a block hash.
pub fn block_hash_from_id(&self, id: &BlockId) -> error::Result<Option<block::HeaderHash>> {
match *id {
BlockId::Hash(h) => Ok(Some(h)),
BlockId::Number(n) => self.block_hash(n),
}
}
/// Convert an arbitrary block ID into a block hash.
pub fn block_number_from_id(&self, id: &BlockId) -> error::Result<Option<block::Number>> {
match *id {
BlockId::Hash(_) => Ok(self.header(id)?.map(|h| h.number)),
BlockId::Number(n) => Ok(Some(n)),
}
}
/// Get block header by id.
pub fn header(&self, id: &BlockId) -> error::Result<Option<block::Header>> {
self.backend.blockchain().header(*id)
@@ -254,11 +313,38 @@ mod tests {
use super::*;
use codec::Slicable;
use keyring::Keyring;
use primitives::block::Transaction as PrimitiveTransaction;
use test_runtime::genesismap::{GenesisConfig, additional_storage_with_genesis};
use test_runtime::{UncheckedTransaction, Transaction};
use test_runtime;
native_executor_instance!(Executor, test_runtime::api::dispatch, include_bytes!("../../test-runtime/wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.compact.wasm"));
fn genesis_config() -> GenesisConfig {
GenesisConfig::new_simple(vec![
Keyring::Alice.to_raw_public(),
Keyring::Bob.to_raw_public(),
Keyring::Charlie.to_raw_public()
], 1000)
}
fn prepare_genesis() -> (primitives::block::Header, Vec<(Vec<u8>, Vec<u8>)>) {
let mut storage = genesis_config().genesis_map();
let block = genesis::construct_genesis_block(&storage);
storage.extend(additional_storage_with_genesis(&block));
(primitives::block::Header::decode(&mut block.header.encode().as_ref()).expect("to_vec() always gives a valid serialisation; qed"), storage.into_iter().collect())
}
#[test]
fn client_initialises_from_genesis_ok() {
let client = new_in_mem(Executor::new(), prepare_genesis).unwrap();
let genesis_hash = client.block_hash(0).unwrap().unwrap();
assert_eq!(client.using_environment(|| test_runtime::system::latest_block_hash()).unwrap(), genesis_hash);
assert_eq!(client.using_environment(|| test_runtime::system::balance_of(Keyring::Alice.to_raw_public())).unwrap(), 1000);
assert_eq!(client.using_environment(|| test_runtime::system::balance_of(Keyring::Ferdie.to_raw_public())).unwrap(), 0);
}
#[test]
fn authorities_call_works() {
let genesis_config = GenesisConfig::new_simple(vec![
@@ -275,10 +361,79 @@ mod tests {
};
let client = new_in_mem(Executor::new(), prepare_genesis).unwrap();
assert_eq!(client.info().unwrap().chain.best_number, 0);
assert_eq!(client.authorities_at(&BlockId::Number(0)).unwrap(), vec![
Keyring::Alice.to_raw_public(),
Keyring::Bob.to_raw_public(),
Keyring::Charlie.to_raw_public()
]);
}
#[test]
fn block_builder_works_with_no_transactions() {
let genesis_config = GenesisConfig::new_simple(vec![
Keyring::Alice.to_raw_public(),
Keyring::Bob.to_raw_public(),
Keyring::Charlie.to_raw_public()
], 1000);
let prepare_genesis = || {
let mut storage = genesis_config.genesis_map();
let block = genesis::construct_genesis_block(&storage);
storage.extend(additional_storage_with_genesis(&block));
(primitives::block::Header::decode(&mut block.header.encode().as_ref()).expect("to_vec() always gives a valid serialisation; qed"), storage.into_iter().collect())
};
let client = new_in_mem(Executor::new(), prepare_genesis).unwrap();
let builder = client.new_block().unwrap();
let block = builder.bake().unwrap();
client.import_block(block.header, Some(block.transactions)).unwrap();
assert_eq!(client.info().unwrap().chain.best_number, 1);
assert_eq!(client.using_environment(|| test_runtime::system::latest_block_hash()).unwrap(), client.block_hash(1).unwrap().unwrap());
}
trait Signable {
fn signed(self) -> PrimitiveTransaction;
}
impl Signable for Transaction {
fn signed(self) -> PrimitiveTransaction {
let signature = Keyring::from_raw_public(self.from.clone()).unwrap().sign(&self.encode());
PrimitiveTransaction::decode(&mut UncheckedTransaction { signature, tx: self }.encode().as_ref()).unwrap()
}
}
#[test]
fn block_builder_works_with_transactions() {
let genesis_config = GenesisConfig::new_simple(vec![
Keyring::Alice.to_raw_public(),
Keyring::Bob.to_raw_public(),
Keyring::Charlie.to_raw_public()
], 1000);
let prepare_genesis = || {
let mut storage = genesis_config.genesis_map();
let block = genesis::construct_genesis_block(&storage);
storage.extend(additional_storage_with_genesis(&block));
(primitives::block::Header::decode(&mut block.header.encode().as_ref()).expect("to_vec() always gives a valid serialisation; qed"), storage.into_iter().collect())
};
let client = new_in_mem(Executor::new(), prepare_genesis).unwrap();
let mut builder = client.new_block().unwrap();
builder.push(Transaction {
from: Keyring::Alice.to_raw_public(),
to: Keyring::Ferdie.to_raw_public(),
amount: 42,
nonce: 0
}.signed()).unwrap();
let block = builder.bake().unwrap();
client.import_block(block.header, Some(block.transactions)).unwrap();
assert_eq!(client.info().unwrap().chain.best_number, 1);
assert!(client.state_at(&BlockId::Number(1)).unwrap() != client.state_at(&BlockId::Number(0)).unwrap());
assert_eq!(client.using_environment(|| test_runtime::system::balance_of(Keyring::Alice.to_raw_public())).unwrap(), 958);
assert_eq!(client.using_environment(|| test_runtime::system::balance_of(Keyring::Ferdie.to_raw_public())).unwrap(), 42);
}
}