mirror of
https://github.com/pezkuwichain/pezkuwi-subxt.git
synced 2026-06-20 14:01:02 +00:00
import-blocks: Do not read stdin to memory (#11072)
* import-blocks: Do not read `stdin` to memory This fixes a bug with `import-blocks` reading the entire `stdin` before starting to import the blocks. However, for huge files that uses quite a lot of memory. We can just read from `stdin` step by step as we do it with a file. This ensures that we don't read the entire input at once into memory. * FMT * Fix warning
This commit is contained in:
@@ -72,13 +72,9 @@ impl ImportBlocksCmd {
|
|||||||
B: BlockT + for<'de> serde::Deserialize<'de>,
|
B: BlockT + for<'de> serde::Deserialize<'de>,
|
||||||
IQ: sc_service::ImportQueue<B> + 'static,
|
IQ: sc_service::ImportQueue<B> + 'static,
|
||||||
{
|
{
|
||||||
let file: Box<dyn ReadPlusSeek + Send> = match &self.input {
|
let file: Box<dyn Read + Send> = match &self.input {
|
||||||
Some(filename) => Box::new(fs::File::open(filename)?),
|
Some(filename) => Box::new(fs::File::open(filename)?),
|
||||||
None => {
|
None => Box::new(io::stdin()),
|
||||||
let mut buffer = Vec::new();
|
|
||||||
io::stdin().read_to_end(&mut buffer)?;
|
|
||||||
Box::new(io::Cursor::new(buffer))
|
|
||||||
},
|
|
||||||
};
|
};
|
||||||
|
|
||||||
import_blocks(client, import_queue, file, false, self.binary)
|
import_blocks(client, import_queue, file, false, self.binary)
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ use sp_runtime::{
|
|||||||
};
|
};
|
||||||
use std::{
|
use std::{
|
||||||
convert::{TryFrom, TryInto},
|
convert::{TryFrom, TryInto},
|
||||||
io::{Read, Seek},
|
io::Read,
|
||||||
pin::Pin,
|
pin::Pin,
|
||||||
task::Poll,
|
task::Poll,
|
||||||
time::{Duration, Instant},
|
time::{Duration, Instant},
|
||||||
@@ -63,7 +63,7 @@ pub fn build_spec(spec: &dyn ChainSpec, raw: bool) -> error::Result<String> {
|
|||||||
/// SignedBlock and return it.
|
/// SignedBlock and return it.
|
||||||
enum BlockIter<R, B>
|
enum BlockIter<R, B>
|
||||||
where
|
where
|
||||||
R: std::io::Read + std::io::Seek,
|
R: std::io::Read,
|
||||||
{
|
{
|
||||||
Binary {
|
Binary {
|
||||||
// Total number of blocks we are expecting to decode.
|
// Total number of blocks we are expecting to decode.
|
||||||
@@ -83,7 +83,7 @@ where
|
|||||||
|
|
||||||
impl<R, B> BlockIter<R, B>
|
impl<R, B> BlockIter<R, B>
|
||||||
where
|
where
|
||||||
R: Read + Seek + 'static,
|
R: Read + 'static,
|
||||||
B: BlockT + MaybeSerializeDeserialize,
|
B: BlockT + MaybeSerializeDeserialize,
|
||||||
{
|
{
|
||||||
fn new(input: R, binary: bool) -> Result<Self, String> {
|
fn new(input: R, binary: bool) -> Result<Self, String> {
|
||||||
@@ -119,7 +119,7 @@ where
|
|||||||
|
|
||||||
impl<R, B> Iterator for BlockIter<R, B>
|
impl<R, B> Iterator for BlockIter<R, B>
|
||||||
where
|
where
|
||||||
R: Read + Seek + 'static,
|
R: Read + 'static,
|
||||||
B: BlockT + MaybeSerializeDeserialize,
|
B: BlockT + MaybeSerializeDeserialize,
|
||||||
{
|
{
|
||||||
type Item = Result<SignedBlock<B>, String>;
|
type Item = Result<SignedBlock<B>, String>;
|
||||||
@@ -267,10 +267,11 @@ impl<B: BlockT> Speedometer<B> {
|
|||||||
/// Different State that the `import_blocks` future could be in.
|
/// Different State that the `import_blocks` future could be in.
|
||||||
enum ImportState<R, B>
|
enum ImportState<R, B>
|
||||||
where
|
where
|
||||||
R: Read + Seek + 'static,
|
R: Read + 'static,
|
||||||
B: BlockT + MaybeSerializeDeserialize,
|
B: BlockT + MaybeSerializeDeserialize,
|
||||||
{
|
{
|
||||||
/// We are reading from the BlockIter structure, adding those blocks to the queue if possible.
|
/// We are reading from the [`BlockIter`] structure, adding those blocks to the queue if
|
||||||
|
/// possible.
|
||||||
Reading { block_iter: BlockIter<R, B> },
|
Reading { block_iter: BlockIter<R, B> },
|
||||||
/// The queue is full (contains at least MAX_PENDING_BLOCKS blocks) and we are waiting for it
|
/// The queue is full (contains at least MAX_PENDING_BLOCKS blocks) and we are waiting for it
|
||||||
/// to catch up.
|
/// to catch up.
|
||||||
@@ -291,7 +292,7 @@ where
|
|||||||
pub fn import_blocks<B, IQ, C>(
|
pub fn import_blocks<B, IQ, C>(
|
||||||
client: Arc<C>,
|
client: Arc<C>,
|
||||||
mut import_queue: IQ,
|
mut import_queue: IQ,
|
||||||
input: impl Read + Seek + Send + 'static,
|
input: impl Read + Send + 'static,
|
||||||
force: bool,
|
force: bool,
|
||||||
binary: bool,
|
binary: bool,
|
||||||
) -> Pin<Box<dyn Future<Output = Result<(), Error>> + Send>>
|
) -> Pin<Box<dyn Future<Output = Result<(), Error>> + Send>>
|
||||||
|
|||||||
Reference in New Issue
Block a user