Fetching multiple remote values using single message (#3612)

* multiple remote values in single message

* keys_str: String ->keys_str: closure

substrate/core/network/src/chain.rs

@@ -49,10 +49,15 @@ pub trait Client<Block: BlockT>: Send + Sync {
 	fn header_proof(&self, block_number: <Block::Header as HeaderT>::Number) -> Result<(Block::Header, Vec<Vec<u8>>), Error>;
 
 	/// Get storage read execution proof.
-	fn read_proof(&self, block: &Block::Hash, key: &[u8]) -> Result<Vec<Vec<u8>>, Error>;
+	fn read_proof(&self, block: &Block::Hash, keys: &[Vec<u8>]) -> Result<Vec<Vec<u8>>, Error>;
 
 	/// Get child storage read execution proof.
-	fn read_child_proof(&self, block: &Block::Hash, storage_key: &[u8], key: &[u8]) -> Result<Vec<Vec<u8>>, Error>;
+	fn read_child_proof(
+		&self,
+		block: &Block::Hash,
+		storage_key: &[u8],
+		keys: &[Vec<u8>],
+	) -> Result<Vec<Vec<u8>>, Error>;
 
 	/// Get method execution proof.
 	fn execution_proof(&self, block: &Block::Hash, method: &str, data: &[u8]) -> Result<(Vec<u8>, Vec<Vec<u8>>), Error>;
@@ -113,18 +118,18 @@ impl<B, E, Block, RA> Client<Block> for SubstrateClient<B, E, Block, RA> where
 		(self as &SubstrateClient<B, E, Block, RA>).header_proof(&BlockId::Number(block_number))
 	}
 
-	fn read_proof(&self, block: &Block::Hash, key: &[u8]) -> Result<Vec<Vec<u8>>, Error> {
-		(self as &SubstrateClient<B, E, Block, RA>).read_proof(&BlockId::Hash(block.clone()), key)
+	fn read_proof(&self, block: &Block::Hash, keys: &[Vec<u8>]) -> Result<Vec<Vec<u8>>, Error> {
+		(self as &SubstrateClient<B, E, Block, RA>).read_proof(&BlockId::Hash(block.clone()), keys)
 	}
 
 	fn read_child_proof(
 		&self,
 		block: &Block::Hash,
 		storage_key: &[u8],
-		key: &[u8]
+		keys: &[Vec<u8>],
 	) -> Result<Vec<Vec<u8>>, Error> {
 		(self as &SubstrateClient<B, E, Block, RA>)
-			.read_child_proof(&BlockId::Hash(block.clone()), storage_key, key)
+			.read_child_proof(&BlockId::Hash(block.clone()), storage_key, keys)
 	}
 
 	fn execution_proof(&self, block: &Block::Hash, method: &str, data: &[u8]) -> Result<(Vec<u8>, Vec<Vec<u8>>), Error> {

substrate/core/network/src/on_demand_layer.rs

@@ -17,6 +17,7 @@
 //! On-demand requests service.
 
 use crate::protocol::light_dispatch::RequestData;
+use std::collections::HashMap;
 use std::sync::Arc;
 use futures::{prelude::*, sync::mpsc, sync::oneshot};
 use futures03::compat::{Compat01As03, Future01CompatExt as _};
@@ -84,7 +85,7 @@ impl<B> Fetcher<B> for OnDemand<B> where
 	B::Header: HeaderT,
 {
 	type RemoteHeaderResult = Compat01As03<RemoteResponse<B::Header>>;
-	type RemoteReadResult = Compat01As03<RemoteResponse<Option<Vec<u8>>>>;
+	type RemoteReadResult = Compat01As03<RemoteResponse<HashMap<Vec<u8>, Option<Vec<u8>>>>>;
 	type RemoteCallResult = Compat01As03<RemoteResponse<Vec<u8>>>;
 	type RemoteChangesResult = Compat01As03<RemoteResponse<Vec<(NumberFor<B>, u32)>>>;
 	type RemoteBodyResult = Compat01As03<RemoteResponse<Vec<B::Extrinsic>>>;

substrate/core/network/src/protocol.rs

@@ -172,11 +172,17 @@ impl<'a, B: BlockT> LightDispatchNetwork<B> for LightDispatchIn<'a, B> {
 		self.behaviour.send_packet(who, message)
 	}
 
-	fn send_read_request(&mut self, who: &PeerId, id: RequestId, block: <B as BlockT>::Hash, key: Vec<u8>) {
+	fn send_read_request(
+		&mut self,
+		who: &PeerId,
+		id: RequestId,
+		block: <B as BlockT>::Hash,
+		keys: Vec<Vec<u8>>,
+	) {
 		let message = message::generic::Message::RemoteReadRequest(message::RemoteReadRequest {
 			id,
 			block,
-			key,
+			keys,
 		});
 
 		self.behaviour.send_packet(who, message)
@@ -188,13 +194,13 @@ impl<'a, B: BlockT> LightDispatchNetwork<B> for LightDispatchIn<'a, B> {
 		id: RequestId,
 		block: <B as BlockT>::Hash,
 		storage_key: Vec<u8>,
-		key: Vec<u8>
+		keys: Vec<Vec<u8>>,
 	) {
 		let message = message::generic::Message::RemoteReadChildRequest(message::RemoteReadChildRequest {
 			id,
 			block,
 			storage_key,
-			key,
+			keys,
 		});
 
 		self.behaviour.send_packet(who, message)
@@ -1272,15 +1278,24 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 		who: PeerId,
 		request: message::RemoteReadRequest<B::Hash>,
 	) {
+		let keys_str = || match request.keys.len() {
+			1 => request.keys[0].to_hex::<String>(),
+			_ => format!(
+				"{}..{}",
+				request.keys[0].to_hex::<String>(),
+				request.keys[request.keys.len() - 1].to_hex::<String>(),
+			),
+		};
+
 		trace!(target: "sync", "Remote read request {} from {} ({} at {})",
-			request.id, who, request.key.to_hex::<String>(), request.block);
-		let proof = match self.context_data.chain.read_proof(&request.block, &request.key) {
+			request.id, who, keys_str(), request.block);
+		let proof = match self.context_data.chain.read_proof(&request.block, &request.keys) {
 			Ok(proof) => proof,
 			Err(error) => {
 				trace!(target: "sync", "Remote read request {} from {} ({} at {}) failed with: {}",
 					request.id,
 					who,
-					request.key.to_hex::<String>(),
+					keys_str(),
 					request.block,
 					error
 				);
@@ -1301,16 +1316,29 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 		who: PeerId,
 		request: message::RemoteReadChildRequest<B::Hash>,
 	) {
+		let keys_str = || match request.keys.len() {
+			1 => request.keys[0].to_hex::<String>(),
+			_ => format!(
+				"{}..{}",
+				request.keys[0].to_hex::<String>(),
+				request.keys[request.keys.len() - 1].to_hex::<String>(),
+			),
+		};
+
 		trace!(target: "sync", "Remote read child request {} from {} ({} {} at {})",
-			request.id, who, request.storage_key.to_hex::<String>(), request.key.to_hex::<String>(), request.block);
-		let proof = match self.context_data.chain.read_child_proof(&request.block, &request.storage_key, &request.key) {
+			request.id, who, request.storage_key.to_hex::<String>(), keys_str(), request.block);
+		let proof = match self.context_data.chain.read_child_proof(
+			&request.block,
+			&request.storage_key,
+			&request.keys,
+		) {
 			Ok(proof) => proof,
 			Err(error) => {
 				trace!(target: "sync", "Remote read child request {} from {} ({} {} at {}) failed with: {}",
 					request.id,
 					who,
 					request.storage_key.to_hex::<String>(),
-					request.key.to_hex::<String>(),
+					keys_str(),
 					request.block,
 					error
 				);

substrate/core/network/src/protocol/light_dispatch.rs

@@ -53,7 +53,13 @@ pub trait LightDispatchNetwork<B: BlockT> {
 	fn send_header_request(&mut self, who: &PeerId, id: RequestId, block: <<B as BlockT>::Header as HeaderT>::Number);
 
 	/// Send to `who` a read request.
-	fn send_read_request(&mut self, who: &PeerId, id: RequestId, block: <B as BlockT>::Hash, key: Vec<u8>);
+	fn send_read_request(
+		&mut self,
+		who: &PeerId,
+		id: RequestId,
+		block: <B as BlockT>::Hash,
+		keys: Vec<Vec<u8>>,
+	);
 
 	/// Send to `who` a child read request.
 	fn send_read_child_request(
@@ -62,7 +68,7 @@ pub trait LightDispatchNetwork<B: BlockT> {
 		id: RequestId,
 		block: <B as BlockT>::Hash,
 		storage_key: Vec<u8>,
-		key: Vec<u8>
+		keys: Vec<Vec<u8>>,
 	);
 
 	/// Send to `who` a call request.
@@ -135,10 +141,13 @@ struct Request<Block: BlockT> {
 pub(crate) enum RequestData<Block: BlockT> {
 	RemoteBody(RemoteBodyRequest<Block::Header>, OneShotSender<Result<Vec<Block::Extrinsic>, ClientError>>),
 	RemoteHeader(RemoteHeaderRequest<Block::Header>, OneShotSender<Result<Block::Header, ClientError>>),
-	RemoteRead(RemoteReadRequest<Block::Header>, OneShotSender<Result<Option<Vec<u8>>, ClientError>>),
+	RemoteRead(
+		RemoteReadRequest<Block::Header>,
+		OneShotSender<Result<HashMap<Vec<u8>, Option<Vec<u8>>>, ClientError>>,
+	),
 	RemoteReadChild(
 		RemoteReadChildRequest<Block::Header>,
-		OneShotSender<Result<Option<Vec<u8>>, ClientError>>
+		OneShotSender<Result<HashMap<Vec<u8>, Option<Vec<u8>>>, ClientError>>
 	),
 	RemoteCall(RemoteCallRequest<Block::Header>, OneShotSender<Result<Vec<u8>, ClientError>>),
 	RemoteChanges(
@@ -174,7 +183,7 @@ impl<Block: BlockT> FetchChecker<Block> for AlwaysBadChecker {
 		&self,
 		_request: &RemoteReadRequest<Block::Header>,
 		_remote_proof: Vec<Vec<u8>>
-	) -> Result<Option<Vec<u8>>, ClientError> {
+	) -> Result<HashMap<Vec<u8>,Option<Vec<u8>>>, ClientError> {
 		Err(ClientError::Msg("AlwaysBadChecker".into()))
 	}
 
@@ -182,7 +191,7 @@ impl<Block: BlockT> FetchChecker<Block> for AlwaysBadChecker {
 		&self,
 		_request: &RemoteReadChildRequest<Block::Header>,
 		_remote_proof: Vec<Vec<u8>>
-	) -> Result<Option<Vec<u8>>, ClientError> {
+	) -> Result<HashMap<Vec<u8>, Option<Vec<u8>>>, ClientError> {
 		Err(ClientError::Msg("AlwaysBadChecker".into()))
 	}
 
@@ -604,7 +613,7 @@ impl<Block: BlockT> Request<Block> {
 					peer,
 					self.id,
 					data.block,
-					data.key.clone(),
+					data.keys.clone(),
 				),
 			RequestData::RemoteReadChild(ref data, _) =>
 				out.send_read_child_request(
@@ -612,7 +621,7 @@ impl<Block: BlockT> Request<Block> {
 					self.id,
 					data.block,
 					data.storage_key.clone(),
-					data.key.clone(),
+					data.keys.clone(),
 				),
 			RequestData::RemoteCall(ref data, _) =>
 				out.send_call_request(
@@ -663,7 +672,7 @@ impl<Block: BlockT> RequestData<Block> {
 
 #[cfg(test)]
 pub mod tests {
-	use std::collections::HashSet;
+	use std::collections::{HashMap, HashSet};
 	use std::sync::Arc;
 	use std::time::Instant;
 	use futures::{Future, sync::oneshot};
@@ -693,20 +702,34 @@ pub mod tests {
 			}
 		}
 
-		fn check_read_proof(&self, _: &RemoteReadRequest<Header>, _: Vec<Vec<u8>>) -> ClientResult<Option<Vec<u8>>> {
+		fn check_read_proof(
+			&self,
+			request: &RemoteReadRequest<Header>,
+			_: Vec<Vec<u8>>,
+		) -> ClientResult<HashMap<Vec<u8>, Option<Vec<u8>>>> {
 			match self.ok {
-				true => Ok(Some(vec![42])),
+				true => Ok(request.keys
+					.iter()
+					.cloned()
+					.map(|k| (k, Some(vec![42])))
+					.collect()
+				),
 				false => Err(ClientError::Backend("Test error".into())),
 			}
 		}
 
 		fn check_read_child_proof(
 			&self,
-			_: &RemoteReadChildRequest<Header>,
+			request: &RemoteReadChildRequest<Header>,
 			_: Vec<Vec<u8>>
-		) -> ClientResult<Option<Vec<u8>>> {
+		) -> ClientResult<HashMap<Vec<u8>, Option<Vec<u8>>>> {
 			match self.ok {
-				true => Ok(Some(vec![42])),
+				true => Ok(request.keys
+					.iter()
+					.cloned()
+					.map(|k| (k, Some(vec![42])))
+					.collect()
+				),
 				false => Err(ClientError::Backend("Test error".into())),
 			}
 		}
@@ -782,9 +805,9 @@ pub mod tests {
 			self.disconnected_peers.insert(who.clone());
 		}
 		fn send_header_request(&mut self, _: &PeerId, _: RequestId, _: <<B as BlockT>::Header as HeaderT>::Number) {}
-		fn send_read_request(&mut self, _: &PeerId, _: RequestId, _: <B as BlockT>::Hash, _: Vec<u8>) {}
+		fn send_read_request(&mut self, _: &PeerId, _: RequestId, _: <B as BlockT>::Hash, _: Vec<Vec<u8>>) {}
 		fn send_read_child_request(&mut self, _: &PeerId, _: RequestId, _: <B as BlockT>::Hash, _: Vec<u8>,
-			_: Vec<u8>) {}
+			_: Vec<Vec<u8>>) {}
 		fn send_call_request(&mut self, _: &PeerId, _: RequestId, _: <B as BlockT>::Hash, _: String, _: Vec<u8>) {}
 		fn send_changes_request(&mut self, _: &PeerId, _: RequestId, _: <B as BlockT>::Hash, _: <B as BlockT>::Hash,
 			_: <B as BlockT>::Hash, _: <B as BlockT>::Hash, _: Option<Vec<u8>>, _: Vec<u8>) {}
@@ -984,7 +1007,7 @@ pub mod tests {
 		light_dispatch.add_request(&mut network_interface, RequestData::RemoteRead(RemoteReadRequest {
 			header: dummy_header(),
 			block: Default::default(),
-			key: b":key".to_vec(),
+			keys: vec![b":key".to_vec()],
 			retry_count: None,
 		}, tx));
 
@@ -992,7 +1015,7 @@ pub mod tests {
 			id: 0,
 			proof: vec![vec![2]],
 		});
-		assert_eq!(response.wait().unwrap().unwrap(), Some(vec![42]));
+		assert_eq!(response.wait().unwrap().unwrap().remove(b":key".as_ref()).unwrap(), Some(vec![42]));
 	}
 
 	#[test]
@@ -1007,7 +1030,7 @@ pub mod tests {
 			header: dummy_header(),
 			block: Default::default(),
 			storage_key: b":child_storage:sub".to_vec(),
-			key: b":key".to_vec(),
+			keys: vec![b":key".to_vec()],
 			retry_count: None,
 		}, tx));
 
@@ -1016,7 +1039,7 @@ pub mod tests {
 				id: 0,
 				proof: vec![vec![2]],
 		});
-		assert_eq!(response.wait().unwrap().unwrap(), Some(vec![42]));
+		assert_eq!(response.wait().unwrap().unwrap().remove(b":key".as_ref()).unwrap(), Some(vec![42]));
 	}
 
 	#[test]

substrate/core/network/src/protocol/message.rs

@@ -284,7 +284,7 @@ pub mod generic {
 		/// Block at which to perform call.
 		pub block: H,
 		/// Storage key.
-		pub key: Vec<u8>,
+		pub keys: Vec<Vec<u8>>,
 	}
 
 	#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
@@ -297,7 +297,7 @@ pub mod generic {
 		/// Child Storage key.
 		pub storage_key: Vec<u8>,
 		/// Storage key.
-		pub key: Vec<u8>,
+		pub keys: Vec<Vec<u8>>,
 	}
 
 	#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]