Block import and export (#272)

* Block export and import

* Export and import using std streams

* Made AuthorituId::from_slice private
This commit is contained in:
Arkadiy Paronyan
2018-07-03 20:20:53 +02:00
committed by Gav Wood
parent 12268ae700
commit aa747e3fae
10 changed files with 267 additions and 25 deletions
+9 -8
View File
@@ -168,14 +168,14 @@ dependencies = [
[[package]]
name = "clap"
version = "2.31.2"
version = "2.32.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
"atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
"bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
"strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"yaml-rust 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -301,7 +301,7 @@ dependencies = [
name = "demo-cli"
version = "0.1.0"
dependencies = [
"clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
"ctrlc 1.1.1 (git+https://github.com/paritytech/rust-ctrlc.git)",
"demo-executor 0.1.0",
"demo-primitives 0.1.0",
@@ -1401,7 +1401,7 @@ dependencies = [
"ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
"app_dirs 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
"ctrlc 1.1.1 (git+https://github.com/paritytech/rust-ctrlc.git)",
"ed25519 0.1.0",
"env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1420,6 +1420,7 @@ dependencies = [
"serde_json 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)",
"slog 2.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
"substrate-client 0.1.0",
"substrate-codec 0.1.0",
"substrate-network 0.1.0",
"substrate-primitives 0.1.0",
"substrate-rpc 0.1.0",
@@ -1538,7 +1539,7 @@ dependencies = [
name = "polkadot-service"
version = "0.2.0"
dependencies = [
"clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
"ed25519 0.1.0",
"error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
"exit-future 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2687,7 +2688,7 @@ dependencies = [
[[package]]
name = "textwrap"
version = "0.9.0"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3249,7 +3250,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum cc 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)" = "49ec142f5768efb5b7622aebc3fdbdbb8950a4b9ba996393cb76ef7466e8747d"
"checksum cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "405216fd8fe65f718daa7102ea808a946b6ce40c742998fbfd3463645552de18"
"checksum chrono 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1cce36c92cb605414e9b824f866f5babe0a0368e39ea07393b9b63cf3844c0e6"
"checksum clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f0f16b89cbb9ee36d87483dc939fe9f1e13c05898d56d7b230a0d4dff033a536"
"checksum clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b957d88f4b6a63b9d70d5f454ac8011819c6efa7727858f458ab71c756ce2d3e"
"checksum cmake 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)" = "95470235c31c726d72bf2e1f421adc1e65b9d561bf5529612cbe1a72da1467b3"
"checksum constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ff012e225ce166d4422e0e78419d901719760f62ae2b7969ca6b564d1b54a9e"
"checksum core-foundation 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "25bfd746d203017f7d5cbd31ee5d8e17f94b6521c7af77ece6c9e4b2d4b16c67"
@@ -3431,7 +3432,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum take_mut 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60"
"checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096"
"checksum textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b59b6b4b44d867f1370ef1bd91bfb262bf07bf0ae65c202ea2fbc16153b693"
"checksum textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "307686869c93e71f94da64286f9a9524c0f308a9e1c87a583de8e9c9039ad3f6"
"checksum thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "279ef31c19ededf577bfd12dfae728040a21f635b06a24cd670ff510edd38963"
"checksum time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "d825be0eb33fda1a7e68012d51e9c7f451dc1a69391e7fdc197060bb8c56667b"
"checksum timer 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "31d42176308937165701f50638db1c31586f183f1aab416268216577aec7306b"
+1
View File
@@ -26,6 +26,7 @@ parking_lot = "0.4"
serde_json = "1.0"
serde = "1.0"
substrate-client = { path = "../../substrate/client" }
substrate-codec = { path = "../../substrate/codec" }
substrate-network = { path = "../../substrate/network" }
substrate-primitives = { path = "../../substrate/primitives" }
substrate-rpc = { path = "../../substrate/rpc" }
+50
View File
@@ -104,3 +104,53 @@ subcommands:
value_name: CHAIN_SPEC
help: Specify the chain specification (one of dev, local or poc-2)
takes_value: true
- export-blocks:
about: Export blocks to a file
args:
- OUTPUT:
index: 1
help: Output file name or stdout if unspecified.
required: false
- chain:
long: chain
value_name: CHAIN_SPEC
help: Specify the chain specification.
takes_value: true
- base-path:
long: base-path
short: d
value_name: PATH
help: Specify custom base path.
takes_value: true
- from:
long: from
value_name: BLOCK
help: Specify starting block number. 1 by default.
takes_value: true
- to:
long: to
value_name: BLOCK
help: Specify last block number. Best block by default.
takes_value: true
- json:
long: json
help: Use JSON output rather than binary.
takes_value: false
- import-blocks:
about: Import blocks from file.
args:
- INPUT:
index: 1
help: Input file or stdin if unspecified.
required: false
- chain:
long: chain
value_name: CHAIN_SPEC
help: Specify the chain specification.
takes_value: true
- base-path:
long: base-path
short: d
value_name: PATH
help: Specify custom base path.
takes_value: true
+138 -13
View File
@@ -36,6 +36,7 @@ extern crate serde_json;
extern crate substrate_client as client;
extern crate substrate_network as network;
extern crate substrate_codec as codec;
extern crate substrate_primitives;
extern crate substrate_rpc;
extern crate substrate_rpc_servers as rpc;
@@ -65,11 +66,15 @@ mod chain_spec;
pub use chain_spec::ChainSpec;
use std::io;
use std::io::{self, Write, Read, stdin, stdout};
use std::fs::File;
use std::net::SocketAddr;
use std::path::{Path, PathBuf};
use substrate_telemetry::{init_telemetry, TelemetryConfig};
use polkadot_primitives::Block;
use polkadot_primitives::{Block, BlockId};
use codec::Slicable;
use client::BlockOrigin;
use runtime_primitives::generic::SignedBlock;
use futures::sync::mpsc;
use futures::{Sink, Future, Stream};
@@ -106,6 +111,12 @@ fn load_spec(matches: &clap::ArgMatches) -> Result<service::ChainSpec, String> {
Ok(spec)
}
fn base_path(matches: &clap::ArgMatches) -> PathBuf {
matches.value_of("base-path")
.map(|x| Path::new(x).to_owned())
.unwrap_or_else(default_base_path)
}
/// Parse command line arguments and start the node.
///
/// IANA unassigned port ranges that we could use:
@@ -123,10 +134,10 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
Ok(m) => m,
Err(ref e) if e.kind == clap::ErrorKind::VersionDisplayed => return Ok(()),
Err(ref e) if e.kind == clap::ErrorKind::HelpDisplayed => {
let _ = clap::App::from_yaml(yaml).print_long_help();
print!("{}", e);
return Ok(())
}
Err(e) => return Err(e.into()),
Err(e) => e.exit(),
};
// TODO [ToDr] Split parameters parsing from actual execution.
@@ -139,11 +150,15 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
info!(" by Parity Technologies, 2017, 2018");
if let Some(matches) = matches.subcommand_matches("build-spec") {
let spec = load_spec(&matches)?;
info!("Building chain spec");
let json = spec.to_json(matches.is_present("raw"))?;
print!("{}", json);
return Ok(())
return build_spec(matches);
}
if let Some(matches) = matches.subcommand_matches("export-blocks") {
return export_blocks(matches);
}
if let Some(matches) = matches.subcommand_matches("import-blocks") {
return import_blocks(matches);
}
let spec = load_spec(&matches)?;
@@ -154,10 +169,7 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
info!("Node name: {}", config.name);
}
let base_path = matches.value_of("base-path")
.map(|x| Path::new(x).to_owned())
.unwrap_or_else(default_base_path);
let base_path = base_path(&matches);
config.keystore_path = matches.value_of("keystore")
.map(|x| Path::new(x).to_owned())
.unwrap_or_else(|| keystore_path(&base_path))
@@ -245,6 +257,119 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
}
}
fn build_spec(matches: &clap::ArgMatches) -> error::Result<()> {
let spec = load_spec(&matches)?;
info!("Building chain spec");
let json = spec.to_json(matches.is_present("raw"))?;
print!("{}", json);
Ok(())
}
fn export_blocks(matches: &clap::ArgMatches) -> error::Result<()> {
let base_path = base_path(matches);
let spec = load_spec(&matches)?;
let mut config = service::Configuration::default_with_spec(spec);
config.database_path = db_path(&base_path).to_string_lossy().into();
info!("DB path: {}", config.database_path);
let client = service::new_client(config)?;
let (exit_send, exit) = std::sync::mpsc::channel();
ctrlc::CtrlC::set_handler(move || {
exit_send.clone().send(()).expect("Error sending exit notification");
});
info!("Exporting blocks");
let mut block: u32 = match matches.value_of("from") {
Some(v) => v.parse().map_err(|_| "Invalid --from argument")?,
None => 1,
};
let last = match matches.value_of("to") {
Some(v) => v.parse().map_err(|_| "Invalid --to argument")?,
None => client.info()?.chain.best_number as u32,
};
if last < block {
return Err("Invalid block range specified".into());
}
let json = matches.is_present("json");
let mut file: Box<Write> = match matches.value_of("OUTPUT") {
Some(filename) => Box::new(File::open(filename)?),
None => Box::new(stdout()),
};
if !json {
file.write(&(last - block + 1).encode())?;
}
loop {
if exit.try_recv().is_ok() {
break;
}
match client.block(&BlockId::number(block as u64))? {
Some(block) => {
if json {
serde_json::to_writer(&mut *file, &block).map_err(|e| format!("Eror writing JSON: {}", e))?;
} else {
file.write(&block.encode())?;
}
},
None => break,
}
if block % 10000 == 0 {
info!("#{}", block);
}
if block == last {
break;
}
block += 1;
}
Ok(())
}
fn import_blocks(matches: &clap::ArgMatches) -> error::Result<()> {
let spec = load_spec(&matches)?;
let base_path = base_path(matches);
let mut config = service::Configuration::default_with_spec(spec);
config.database_path = db_path(&base_path).to_string_lossy().into();
let client = service::new_client(config)?;
let (exit_send, exit) = std::sync::mpsc::channel();
ctrlc::CtrlC::set_handler(move || {
exit_send.clone().send(()).expect("Error sending exit notification");
});
let mut file: Box<Read> = match matches.value_of("INPUT") {
Some(filename) => Box::new(File::open(filename)?),
None => Box::new(stdin()),
};
info!("Importing blocks");
let count: u32 = Slicable::decode(&mut file).ok_or("Error reading file")?;
let mut block = 0;
for _ in 0 .. count {
if exit.try_recv().is_ok() {
break;
}
match SignedBlock::decode(&mut file) {
Some(block) => {
let header = client.check_justification(block.block.header, block.justification.into())?;
client.import_block(BlockOrigin::File, header, Some(block.block.extrinsics))?;
},
None => {
warn!("Error reading block data.");
break;
}
}
block += 1;
if block % 10000 == 0 {
info!("#{}", block);
}
}
info!("Imported {} blocks. Best: #{}", block, client.info()?.chain.best_number);
Ok(())
}
fn run_until_exit<C>(mut core: reactor::Core, service: service::Service<C>, matches: &clap::ArgMatches, sys_conf: SystemConfiguration) -> error::Result<()>
where
C: service::Components,
+19
View File
@@ -99,6 +99,25 @@ pub fn new_full(config: Configuration) -> Result<Service<components::FullCompone
Service::new(components::FullComponents { is_validator }, config)
}
/// Creates bare client without any networking.
pub fn new_client(config: Configuration) -> Result<Arc<Client<
<components::FullComponents as Components>::Backend,
<components::FullComponents as Components>::Executor,
Block>>,
error::Error>
{
let db_settings = client_db::DatabaseSettings {
cache_size: None,
path: config.database_path.into(),
pruning: config.pruning,
};
let executor = polkadot_executor::Executor::new();
let is_validator = (config.roles & Role::VALIDATOR) == Role::VALIDATOR;
let components = components::FullComponents { is_validator };
let (client, _) = components.build_client(db_settings, executor, &config.chain_spec)?;
Ok(client)
}
impl<Components> Service<Components>
where
Components: components::Components,
+10 -1
View File
@@ -20,7 +20,7 @@ use std::sync::Arc;
use futures::sync::mpsc;
use parking_lot::{Mutex, RwLock};
use primitives::AuthorityId;
use runtime_primitives::{bft::Justification, generic::BlockId};
use runtime_primitives::{bft::Justification, generic::{BlockId, SignedBlock, Block as RuntimeBlock}};
use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Zero, One};
use runtime_primitives::BuildStorage;
use primitives::storage::{StorageKey, StorageData};
@@ -416,6 +416,15 @@ impl<B, E, Block> Client<B, E, Block> where
self.backend.blockchain().justification(*id)
}
/// Get full block by id.
pub fn block(&self, id: &BlockId<Block>) -> error::Result<Option<SignedBlock<Block::Header, Block::Extrinsic, Block::Hash>>> {
Ok(match (self.header(id)?, self.body(id)?, self.justification(id)?) {
(Some(header), Some(extrinsics), Some(justification)) =>
Some(SignedBlock { block: RuntimeBlock { header, extrinsics }, justification }),
_ => None,
})
}
/// Get best block header.
pub fn best_block_header(&self) -> error::Result<<Block as BlockT>::Header> {
let info = self.backend.blockchain().info().map_err(|e| error::Error::from_blockchain(Box::new(e)))?;
+1 -1
View File
@@ -53,7 +53,7 @@ pub use client::{
new_in_mem,
BlockStatus, BlockOrigin, BlockchainEventStream, BlockchainEvents,
Client, ClientInfo, ChainHead,
ImportResult,
ImportResult, JustifiedHeader,
};
pub use blockchain::Info as ChainInfo;
pub use call_executor::{CallResult, CallExecutor, LocalCallExecutor};
@@ -38,6 +38,7 @@ pub trait Input {
}
}
#[cfg(not(feature = "std"))]
impl<'a> Input for &'a [u8] {
fn read(&mut self, into: &mut [u8]) -> usize {
let len = ::core::cmp::min(into.len(), self.len());
@@ -47,6 +48,13 @@ impl<'a> Input for &'a [u8] {
}
}
#[cfg(feature = "std")]
impl<R: ::std::io::Read> Input for R {
fn read(&mut self, into: &mut [u8]) -> usize {
(self as &mut ::std::io::Read).read(into).unwrap_or(0)
}
}
/// Trait that allows zero-copy read/write of value-references to/from slices in LE format.
pub trait Slicable: Sized {
/// Attempt to deserialise the value from input.
@@ -25,8 +25,8 @@ use H256;
pub struct AuthorityId(pub [u8; 32]);
impl AuthorityId {
/// Create an id from byte slice.
pub fn from_slice(data: &[u8]) -> Self {
/// Create an id from a 32-byte slice. Panics with other lengths.
fn from_slice(data: &[u8]) -> Self {
let mut r = [0u8; 32];
r.copy_from_slice(data);
AuthorityId(r)
@@ -28,6 +28,7 @@ use runtime_support::AuxDispatchable;
use traits::{self, Member, SimpleArithmetic, SimpleBitOps, MaybeDisplay, Block as BlockT,
Header as HeaderT, Hashing as HashingT};
use rstd::ops;
use bft::Justification;
/// Definition of something that the external world might want to say.
#[derive(PartialEq, Eq, Clone)]
@@ -476,6 +477,34 @@ where
}
}
/// Abstraction over a substrate block and justification.
#[derive(PartialEq, Eq, Clone)]
#[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))]
#[cfg_attr(feature = "std", serde(rename_all = "camelCase"))]
#[cfg_attr(feature = "std", serde(deny_unknown_fields))]
pub struct SignedBlock<Header, Extrinsic, Hash> {
/// Full block.
pub block: Block<Header, Extrinsic>,
/// Block header justification.
pub justification: Justification<Hash>,
}
impl<Header: Slicable, Extrinsic: Slicable, Hash: Slicable> Slicable for SignedBlock<Header, Extrinsic, Hash> {
fn decode<I: Input>(input: &mut I) -> Option<Self> {
Some(SignedBlock {
block: Slicable::decode(input)?,
justification: Slicable::decode(input)?,
})
}
fn encode(&self) -> Vec<u8> {
let mut v: Vec<u8> = Vec::new();
v.extend(self.block.encode());
v.extend(self.justification.encode());
v
}
}
#[cfg(test)]
mod tests {
use codec::Slicable;