import-blocks: Do not read stdin to memory (#11072)

* import-blocks: Do not read `stdin` to memory

This fixes a bug with `import-blocks` reading the entire `stdin` before starting to import the
blocks. However, for huge files that uses quite a lot of memory. We can just read from `stdin` step
by step as we do it with a file. This ensures that we don't read the entire input at once into memory.

* FMT

* Fix warning
This commit is contained in:
Bastian Köcher
2022-03-21 13:15:37 +01:00
committed by GitHub
parent 522e5d13aa
commit 434eaffd4c
2 changed files with 10 additions and 13 deletions
@@ -72,13 +72,9 @@ impl ImportBlocksCmd {
B: BlockT + for<'de> serde::Deserialize<'de>,
IQ: sc_service::ImportQueue<B> + 'static,
{
let file: Box<dyn ReadPlusSeek + Send> = match &self.input {
let file: Box<dyn Read + Send> = match &self.input {
Some(filename) => Box::new(fs::File::open(filename)?),
None => {
let mut buffer = Vec::new();
io::stdin().read_to_end(&mut buffer)?;
Box::new(io::Cursor::new(buffer))
},
None => Box::new(io::stdin()),
};
import_blocks(client, import_queue, file, false, self.binary)
@@ -36,7 +36,7 @@ use sp_runtime::{
};
use std::{
convert::{TryFrom, TryInto},
io::{Read, Seek},
io::Read,
pin::Pin,
task::Poll,
time::{Duration, Instant},
@@ -63,7 +63,7 @@ pub fn build_spec(spec: &dyn ChainSpec, raw: bool) -> error::Result<String> {
/// SignedBlock and return it.
enum BlockIter<R, B>
where
R: std::io::Read + std::io::Seek,
R: std::io::Read,
{
Binary {
// Total number of blocks we are expecting to decode.
@@ -83,7 +83,7 @@ where
impl<R, B> BlockIter<R, B>
where
R: Read + Seek + 'static,
R: Read + 'static,
B: BlockT + MaybeSerializeDeserialize,
{
fn new(input: R, binary: bool) -> Result<Self, String> {
@@ -119,7 +119,7 @@ where
impl<R, B> Iterator for BlockIter<R, B>
where
R: Read + Seek + 'static,
R: Read + 'static,
B: BlockT + MaybeSerializeDeserialize,
{
type Item = Result<SignedBlock<B>, String>;
@@ -267,10 +267,11 @@ impl<B: BlockT> Speedometer<B> {
/// Different State that the `import_blocks` future could be in.
enum ImportState<R, B>
where
R: Read + Seek + 'static,
R: Read + 'static,
B: BlockT + MaybeSerializeDeserialize,
{
/// We are reading from the BlockIter structure, adding those blocks to the queue if possible.
/// We are reading from the [`BlockIter`] structure, adding those blocks to the queue if
/// possible.
Reading { block_iter: BlockIter<R, B> },
/// The queue is full (contains at least MAX_PENDING_BLOCKS blocks) and we are waiting for it
/// to catch up.
@@ -291,7 +292,7 @@ where
pub fn import_blocks<B, IQ, C>(
client: Arc<C>,
mut import_queue: IQ,
input: impl Read + Seek + Send + 'static,
input: impl Read + Send + 'static,
force: bool,
binary: bool,
) -> Pin<Box<dyn Future<Output = Result<(), Error>> + Send>>