Fetching storage proofs by light client (#252)

* storage proofs

* fixed grumbles

* Update lib.rs

substrate/Cargo.lock

@@ -2841,6 +2841,7 @@ dependencies = [
  "linked-hash-map 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "substrate-client 0.1.0",
  "substrate-codec 0.1.0",
  "substrate-codec-derive 0.1.0",

substrate/substrate/client/src/blockchain.rs

@@ -17,22 +17,27 @@
 //! Polkadot blockchain trait
 
 use primitives::AuthorityId;
-use runtime_primitives::traits::{Block as BlockT, Header as HeaderT};
+use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, NumberFor};
 use runtime_primitives::generic::BlockId;
 use runtime_primitives::bft::Justification;
 
-use error::Result;
+use error::{ErrorKind, Result};
 
 /// Blockchain database header backend. Does not perform any validation.
 pub trait HeaderBackend<Block: BlockT>: Send + Sync {
 	/// Get block header. Returns `None` if block is not found.
-	fn header(&self, id: BlockId<Block>) -> Result<Option<<Block as BlockT>::Header>>;
+	fn header(&self, id: BlockId<Block>) -> Result<Option<Block::Header>>;
 	/// Get blockchain info.
 	fn info(&self) -> Result<Info<Block>>;
 	/// Get block status.
 	fn status(&self, id: BlockId<Block>) -> Result<BlockStatus>;
 	/// Get block hash by number. Returns `None` if the header is not in the chain.
-	fn hash(&self, number: <<Block as BlockT>::Header as HeaderT>::Number) -> Result<Option<<<Block as BlockT>::Header as HeaderT>::Hash>>;
+	fn hash(&self, number: NumberFor<Block>) -> Result<Option<Block::Hash>>;
+
+	/// Get block header. Returns `UnknownBlock` error if block is not found.
+	fn expect_header(&self, id: BlockId<Block>) -> Result<Block::Header> {
+		self.header(id)?.ok_or_else(|| ErrorKind::UnknownBlock(format!("{}", id)).into())
+	}
 }
 
 /// Blockchain database backend. Does not perform any validation.

substrate/substrate/client/src/client.rs

@@ -25,7 +25,10 @@ use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Zero, One,
 use runtime_primitives::BuildStorage;
 use primitives::storage::{StorageKey, StorageData};
 use codec::Decode;
-use state_machine::{Ext, OverlayedChanges, Backend as StateBackend, CodeExecutor, ExecutionStrategy, ExecutionManager};
+use state_machine::{
+	Ext, OverlayedChanges, Backend as StateBackend, CodeExecutor,
+	ExecutionStrategy, ExecutionManager, prove_read
+};
 
 use backend::{self, BlockImportOperation};
 use blockchain::{self, Info as ChainInfo, Backend as ChainBackend, HeaderBackend as ChainHeaderBackend};
@@ -247,6 +250,14 @@ impl<B, E, Block> Client<B, E, Block> where
 		&self.executor
 	}
 
+	/// Reads storage value at a given block + key, returning read proof.
+	pub fn read_proof(&self, id: &BlockId<Block>, key: &[u8]) -> error::Result<Vec<Vec<u8>>> {
+		self.state_at(id)
+			.and_then(|state| prove_read(state, key)
+				.map(|(_, proof)| proof)
+				.map_err(Into::into))
+	}
+
 	/// Execute a call to a contract on top of state in a block of given hash
 	/// AND returning execution proof.
 	///

substrate/substrate/client/src/light/backend.rs

@@ -18,6 +18,8 @@
 //! Everything else is requested from full nodes on demand.
 
 use std::sync::{Arc, Weak};
+use futures::{Future, IntoFuture};
+use parking_lot::RwLock;
 
 use primitives::AuthorityId;
 use runtime_primitives::{bft::Justification, generic::BlockId};
@@ -29,7 +31,7 @@ use backend::{Backend as ClientBackend, BlockImportOperation, RemoteBackend};
 use blockchain::HeaderBackend as BlockchainHeaderBackend;
 use error::{Error as ClientError, ErrorKind as ClientErrorKind, Result as ClientResult};
 use light::blockchain::{Blockchain, Storage as BlockchainStorage};
-use light::fetcher::Fetcher;
+use light::fetcher::{Fetcher, RemoteReadRequest};
 
 /// Light client backend.
 pub struct Backend<S, F> {
@@ -37,17 +39,19 @@ pub struct Backend<S, F> {
 }
 
 /// Light block (header and justification) import operation.
-pub struct ImportOperation<Block: BlockT, F> {
+pub struct ImportOperation<Block: BlockT, S, F> {
 	is_new_best: bool,
 	header: Option<Block::Header>,
 	authorities: Option<Vec<AuthorityId>>,
-	_phantom: ::std::marker::PhantomData<F>,
+	_phantom: ::std::marker::PhantomData<(S, F)>,
 }
 
 /// On-demand state.
-pub struct OnDemandState<Block: BlockT, F> {
+pub struct OnDemandState<Block: BlockT, S, F> {
 	fetcher: Weak<F>,
+	blockchain: Weak<Blockchain<S, F>>,
 	block: Block::Hash,
+	cached_header: RwLock<Option<Block::Header>>,
 }
 
 impl<S, F> Backend<S, F> {
@@ -65,11 +69,11 @@ impl<S, F> Backend<S, F> {
 impl<S, F, Block> ClientBackend<Block> for Backend<S, F> where
 	Block: BlockT,
 	S: BlockchainStorage<Block>,
-	F: Fetcher<Block>,
+	F: Fetcher<Block>
 {
-	type BlockImportOperation = ImportOperation<Block, F>;
+	type BlockImportOperation = ImportOperation<Block, S, F>;
 	type Blockchain = Blockchain<S, F>;
-	type State = OnDemandState<Block, F>;
+	type State = OnDemandState<Block, S, F>;
 
 	fn begin_operation(&self, _block: BlockId<Block>) -> ClientResult<Self::BlockImportOperation> {
 		Ok(ImportOperation {
@@ -96,8 +100,10 @@ impl<S, F, Block> ClientBackend<Block> for Backend<S, F> where
 		};
 
 		Ok(OnDemandState {
-			block: block_hash.ok_or_else(|| ClientErrorKind::UnknownBlock(format!("{}", block)))?,
 			fetcher: self.blockchain.fetcher(),
+			blockchain: Arc::downgrade(&self.blockchain),
+			block: block_hash.ok_or_else(|| ClientErrorKind::UnknownBlock(format!("{}", block)))?,
+			cached_header: RwLock::new(None),
 		})
 	}
 
@@ -108,8 +114,13 @@ impl<S, F, Block> ClientBackend<Block> for Backend<S, F> where
 
 impl<S, F, Block> RemoteBackend<Block> for Backend<S, F> where Block: BlockT, S: BlockchainStorage<Block>, F: Fetcher<Block> {}
 
-impl<F, Block> BlockImportOperation<Block> for ImportOperation<Block, F> where Block: BlockT, F: Fetcher<Block> {
-	type State = OnDemandState<Block, F>;
+impl<S, F, Block> BlockImportOperation<Block> for ImportOperation<Block, S, F>
+	where
+		Block: BlockT,
+		S: BlockchainStorage<Block>,
+		F: Fetcher<Block>,
+{
+	type State = OnDemandState<Block, S, F>;
 
 	fn state(&self) -> ClientResult<Option<&Self::State>> {
 		// None means 'locally-stateless' backend
@@ -143,21 +154,32 @@ impl<F, Block> BlockImportOperation<Block> for ImportOperation<Block, F> where B
 	}
 }
 
-impl<Block: BlockT, F> Clone for OnDemandState<Block, F> {
-	fn clone(&self) -> Self {
-		OnDemandState {
-			fetcher: self.fetcher.clone(),
-			block: self.block,
-		}
-	}
-}
-
-impl<Block, F> StateBackend for OnDemandState<Block, F> where Block: BlockT, F: Fetcher<Block> {
+impl<Block, S, F> StateBackend for OnDemandState<Block, S, F>
+	where
+		Block: BlockT,
+		S: BlockchainStorage<Block>,
+		F: Fetcher<Block>,
+{
 	type Error = ClientError;
 	type Transaction = ();
 
-	fn storage(&self, _key: &[u8]) -> ClientResult<Option<Vec<u8>>> {
-		Err(ClientErrorKind::NotAvailableOnLightClient.into()) // TODO: fetch from remote node
+	fn storage(&self, key: &[u8]) -> ClientResult<Option<Vec<u8>>> {
+		let mut header = self.cached_header.read().clone();
+		if header.is_none() {
+			let cached_header = self.blockchain.upgrade()
+				.ok_or_else(|| ClientErrorKind::UnknownBlock(format!("{}", self.block)).into())
+				.and_then(|blockchain| blockchain.expect_header(BlockId::Hash(self.block)))?;
+			header = Some(cached_header.clone());
+			*self.cached_header.write() = Some(cached_header);
+		}
+
+		self.fetcher.upgrade().ok_or(ClientErrorKind::NotAvailableOnLightClient)?
+			.remote_read(RemoteReadRequest {
+				block: self.block,
+				header: header.expect("if block above guarantees that header is_some(); qed"),
+				key: key.to_vec(),
+			})
+			.into_future().wait()
 	}
 
 	fn for_keys_with_prefix<A: FnMut(&[u8])>(&self, _prefix: &[u8], _action: A) {
@@ -175,7 +197,7 @@ impl<Block, F> StateBackend for OnDemandState<Block, F> where Block: BlockT, F:
 	}
 }
 
-impl<Block, F> TryIntoStateTrieBackend for OnDemandState<Block, F> where Block: BlockT, F: Fetcher<Block> {
+impl<Block, S, F> TryIntoStateTrieBackend for OnDemandState<Block, S, F> where Block: BlockT, F: Fetcher<Block> {
 	fn try_into_trie_backend(self) -> Option<StateTrieBackend> {
 		None
 	}
@@ -183,20 +205,55 @@ impl<Block, F> TryIntoStateTrieBackend for OnDemandState<Block, F> where Block:
 
 #[cfg(test)]
 pub mod tests {
-	use futures::future::{ok, FutureResult};
+	use futures::future::{ok, err, FutureResult};
 	use parking_lot::Mutex;
 	use call_executor::CallResult;
+	use executor::NativeExecutionDispatch;
 	use error::Error as ClientError;
-	use test_client::runtime::{Hash, Block};
-	use light::fetcher::{Fetcher, RemoteCallRequest};
+	use test_client::{self, runtime::{Header, Block}};
+	use light::new_fetch_checker;
+	use light::fetcher::{Fetcher, FetchChecker, RemoteCallRequest};
+	use super::*;
 
 	pub type OkCallFetcher = Mutex<CallResult>;
 
 	impl Fetcher<Block> for OkCallFetcher {
+		type RemoteReadResult = FutureResult<Option<Vec<u8>>, ClientError>;
 		type RemoteCallResult = FutureResult<CallResult, ClientError>;
 
-		fn remote_call(&self, _request: RemoteCallRequest<Hash>) -> Self::RemoteCallResult {
+		fn remote_read(&self, _request: RemoteReadRequest<Header>) -> Self::RemoteReadResult {
+			err("Not implemented on test node".into())
+		}
+
+		fn remote_call(&self, _request: RemoteCallRequest<Header>) -> Self::RemoteCallResult {
 			ok((*self.lock()).clone())
 		}
 	}
+
+	#[test]
+	fn storage_read_proof_is_generated_and_checked() {
+		// prepare remote client
+		let remote_client = test_client::new();
+		let remote_block_id = BlockId::Number(0);
+		let remote_block_hash = remote_client.block_hash(0).unwrap().unwrap();
+		let mut remote_block_header = remote_client.header(&remote_block_id).unwrap().unwrap();
+		remote_block_header.state_root = remote_client.state_at(&remote_block_id)
+			.unwrap().storage_root(::std::iter::empty()).0.into();
+
+		// 'fetch' read proof from remote node
+		let authorities_len = remote_client.authorities_at(&remote_block_id).unwrap().len();
+		let remote_read_proof = remote_client.read_proof(&remote_block_id, b":auth:len").unwrap();
+
+		// check remote read proof locally
+		let local_executor = test_client::LocalExecutor::with_heap_pages(8);
+		let local_checker = new_fetch_checker(local_executor);
+		let request = RemoteReadRequest {
+			block: remote_block_hash,
+			header: remote_block_header,
+			key: b":auth:len".to_vec(),
+		};
+		assert_eq!((&local_checker as &FetchChecker<Block>).check_read_proof(
+			&request,
+			remote_read_proof).unwrap().unwrap()[0], authorities_len as u8);
+	}
 }

substrate/substrate/client/src/light/call_executor.rs

@@ -60,9 +60,11 @@ impl<B, F, Block> CallExecutor<Block> for RemoteCallExecutor<B, F>
 			BlockId::Number(number) => self.blockchain.hash(number)?
 				.ok_or_else(|| ClientErrorKind::UnknownBlock(format!("{}", number)))?,
 		};
+		let block_header = self.blockchain.expect_header(id.clone())?;
 
 		self.fetcher.remote_call(RemoteCallRequest {
-			block: block_hash.clone(),
+			block: block_hash,
+			header: block_header,
 			method: method.into(),
 			call_data: call_data.to_vec(),
 		}).into_future().wait()
@@ -97,34 +99,17 @@ impl<B, F, Block> CallExecutor<Block> for RemoteCallExecutor<B, F>
 }
 
 /// Check remote execution proof using given backend.
-pub fn check_execution_proof<Block, B, E>(
-	blockchain: &B,
+pub fn check_execution_proof<Header, E>(
 	executor: &E,
-	request: &RemoteCallRequest<Block::Hash>,
+	request: &RemoteCallRequest<Header>,
 	remote_proof: Vec<Vec<u8>>
 ) -> ClientResult<CallResult>
 	where
-		Block: BlockT,
-		B: ChainBackend<Block>,
+		Header: HeaderT,
 		E: CodeExecutor,
 {
-	let local_header = blockchain.header(BlockId::Hash(request.block))?;
-	let local_header = local_header.ok_or_else(|| ClientErrorKind::UnknownBlock(format!("{}", request.block)))?;
-	let local_state_root = *local_header.state_root();
-	do_check_execution_proof(local_state_root.into(), executor, request, remote_proof)
-}
+	let local_state_root = request.header.state_root();
 
-/// Check remote execution proof using given state root.
-fn do_check_execution_proof<Hash, E>(
-	local_state_root: Hash,
-	executor: &E,
-	request: &RemoteCallRequest<Hash>,
-	remote_proof: Vec<Vec<u8>>,
-) -> ClientResult<CallResult>
-	where
-		Hash: ::std::fmt::Display + ::std::convert::AsRef<[u8]>,
-		E: CodeExecutor,
-{
 	let mut changes = OverlayedChanges::default();
 	let (local_result, _) = execution_proof_check(
 		TrieH256::from_slice(local_state_root.as_ref()).into(),
@@ -156,8 +141,15 @@ mod tests {
 		
 		// check remote execution proof locally
 		let local_executor = test_client::LocalExecutor::with_heap_pages(8);
-		do_check_execution_proof(remote_block_storage_root.into(), &local_executor, &RemoteCallRequest {
+		check_execution_proof(&local_executor, &RemoteCallRequest {
 			block: test_client::runtime::Hash::default(),
+			header: test_client::runtime::Header {
+				state_root: remote_block_storage_root.into(),
+				parent_hash: Default::default(),
+				number: 0,
+				extrinsics_root: Default::default(),
+				digest: Default::default(),
+			},
 			method: "authorities".into(),
 			call_data: vec![],
 		}, remote_execution_proof).unwrap();

substrate/substrate/client/src/light/fetcher.rs

@@ -16,73 +16,106 @@
 
 //! Light client data fetcher. Fetches requested data from remote full nodes.
 
-use std::sync::Arc;
 use futures::IntoFuture;
 
-use runtime_primitives::traits::{Block as BlockT};
-use state_machine::CodeExecutor;
+use runtime_primitives::traits::{Block as BlockT, Header as HeaderT};
+use state_machine::{CodeExecutor, read_proof_check};
 
 use call_executor::CallResult;
 use error::{Error as ClientError, Result as ClientResult};
-use light::blockchain::{Blockchain, Storage as BlockchainStorage};
 use light::call_executor::check_execution_proof;
 
 /// Remote call request.
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
-pub struct RemoteCallRequest<Hash: ::std::fmt::Display> {
+pub struct RemoteCallRequest<Header: HeaderT> {
 	/// Call at state of given block.
-	pub block: Hash,
+	pub block: Header::Hash,
+	/// Head of block at which call is perormed.
+	pub header: Header,
 	/// Method to call.
 	pub method: String,
 	/// Call data.
 	pub call_data: Vec<u8>,
 }
 
+/// Remote storage read request.
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct RemoteReadRequest<Header: HeaderT> {
+	/// Read at state of given block.
+	pub block: Header::Hash,
+	/// Head of block at which read is perormed.
+	pub header: Header,
+	/// Storage key to read.
+	pub key: Vec<u8>,
+}
+
 /// Light client data fetcher. Implementations of this trait must check if remote data
 /// is correct (see FetchedDataChecker) and return already checked data.
 pub trait Fetcher<Block: BlockT>: Send + Sync {
+	/// Remote storage read future.
+	type RemoteReadResult: IntoFuture<Item=Option<Vec<u8>>, Error=ClientError>;
 	/// Remote call result future.
 	type RemoteCallResult: IntoFuture<Item=CallResult, Error=ClientError>;
 
+	/// Fetch remote storage value.
+	fn remote_read(&self, request: RemoteReadRequest<Block::Header>) -> Self::RemoteReadResult;
 	/// Fetch remote call result.
-	fn remote_call(&self, request: RemoteCallRequest<Block::Hash>) -> Self::RemoteCallResult;
+	fn remote_call(&self, request: RemoteCallRequest<Block::Header>) -> Self::RemoteCallResult;
 }
 
 /// Light client remote data checker.
+///
+/// Implementations of this trait should not use any blockchain data except that is
+/// passed to its methods.
 pub trait FetchChecker<Block: BlockT>: Send + Sync {
+	/// Check remote storage read proof.
+	fn check_read_proof(
+		&self,
+		request: &RemoteReadRequest<Block::Header>,
+		remote_proof: Vec<Vec<u8>>
+	) -> ClientResult<Option<Vec<u8>>>;
 	/// Check remote method execution proof.
-	fn check_execution_proof(&self, request: &RemoteCallRequest<Block::Hash>, remote_proof: Vec<Vec<u8>>) -> ClientResult<CallResult>;
+	fn check_execution_proof(
+		&self,
+		request: &RemoteCallRequest<Block::Header>,
+		remote_proof: Vec<Vec<u8>>
+	) -> ClientResult<CallResult>;
 }
 
 /// Remote data checker.
-pub struct LightDataChecker<S, E, F> {
-	blockchain: Arc<Blockchain<S, F>>,
+pub struct LightDataChecker<E> {
 	executor: E,
 }
 
-impl<S, E, F> LightDataChecker<S, E, F> {
+impl<E> LightDataChecker<E> {
 	/// Create new light data checker.
-	pub fn new(blockchain: Arc<Blockchain<S, F>>, executor: E) -> Self {
+	pub fn new(executor: E) -> Self {
 		Self {
-			blockchain,
 			executor,
 		}
 	}
-
-	/// Get blockchain reference.
-	pub fn blockchain(&self) -> &Arc<Blockchain<S, F>> {
-		&self.blockchain
-	}
 }
 
-impl<S, E, F, Block> FetchChecker<Block> for LightDataChecker<S, E, F>
+impl<E, Block> FetchChecker<Block> for LightDataChecker<E>
 	where
 		Block: BlockT,
-		S: BlockchainStorage<Block>,
+		Block::Hash: Into<[u8; 32]>,
 		E: CodeExecutor,
-		F: Fetcher<Block>,
 {
-	fn check_execution_proof(&self, request: &RemoteCallRequest<Block::Hash>, remote_proof: Vec<Vec<u8>>) -> ClientResult<CallResult> {
-		check_execution_proof(&*self.blockchain, &self.executor, request, remote_proof)
+	fn check_read_proof(
+		&self,
+		request: &RemoteReadRequest<Block::Header>,
+		remote_proof: Vec<Vec<u8>>
+	) -> ClientResult<Option<Vec<u8>>> {
+		let local_state_root = request.header.state_root().clone();
+		read_proof_check(local_state_root.into(), remote_proof, &request.key).map_err(Into::into)
+	}
+
+	fn check_execution_proof(
+		&self,
+		request: &RemoteCallRequest<Block::Header>,
+		remote_proof: Vec<Vec<u8>>
+	) -> ClientResult<CallResult> {
+		check_execution_proof(&self.executor, request, remote_proof)
 	}
 }

substrate/substrate/client/src/light/mod.rs

@@ -62,14 +62,11 @@ pub fn new_light<B, S, F, GS>(
 }
 
 /// Create an instance of fetch data checker.
-pub fn new_fetch_checker<B, S, E, F>(
-	blockchain: Arc<Blockchain<S, F>>,
+pub fn new_fetch_checker<E>(
 	executor: E,
-) -> LightDataChecker<S, E, F>
+) -> LightDataChecker<E>
 	where
-		B: BlockT,
-		S: BlockchainStorage<B>,
 		E: CodeExecutor,
 {
-	LightDataChecker::new(blockchain, executor)
+	LightDataChecker::new(executor)
 }

substrate/substrate/network/Cargo.toml

@@ -14,6 +14,7 @@ error-chain = "0.12"
 bitflags = "1.0"
 futures = "0.1.17"
 linked-hash-map = "0.5"
+rustc-hex = "1.0"
 ethcore-io = { git = "https://github.com/paritytech/parity.git" }
 ed25519 = { path = "../../substrate/ed25519" }
 substrate-primitives = { path = "../../substrate/primitives" }

substrate/substrate/network/src/chain.rs

@@ -45,6 +45,9 @@ pub trait Client<Block: BlockT>: Send + Sync {
 	/// Get block justification.
 	fn justification(&self, id: &BlockId<Block>) -> Result<Option<Justification<Block::Hash>>, Error>;
 
+	/// Get storage read execution proof.
+	fn read_proof(&self, block: &Block::Hash, key: &[u8]) -> Result<Vec<Vec<u8>>, Error>;
+
 	/// Get method execution proof.
 	fn execution_proof(&self, block: &Block::Hash, method: &str, data: &[u8]) -> Result<(Vec<u8>, Vec<Vec<u8>>), Error>;
 }
@@ -84,6 +87,10 @@ impl<B, E, Block> Client<Block> for SubstrateClient<B, E, Block> where
 		(self as &SubstrateClient<B, E, Block>).justification(id)
 	}
 
+	fn read_proof(&self, block: &Block::Hash, key: &[u8]) -> Result<Vec<Vec<u8>>, Error> {
+		(self as &SubstrateClient<B, E, Block>).read_proof(&BlockId::Hash(block.clone()), key)
+	}
+
 	fn execution_proof(&self, block: &Block::Hash, method: &str, data: &[u8]) -> Result<(Vec<u8>, Vec<Vec<u8>>), Error> {
 		(self as &SubstrateClient<B, E, Block>).execution_proof(&BlockId::Hash(block.clone()), method, data)
 	}

substrate/substrate/network/src/lib.rs

@@ -30,6 +30,7 @@ extern crate substrate_network_libp2p as network_libp2p;
 extern crate substrate_codec as codec;
 extern crate futures;
 extern crate ed25519;
+extern crate rustc_hex;
 #[macro_use] extern crate log;
 #[macro_use] extern crate bitflags;
 #[macro_use] extern crate error_chain;
@@ -64,4 +65,4 @@ pub use network_libp2p::{NonReservedPeerMode, NetworkConfiguration, NodeIndex, P
 pub use message::{generic as generic_message, RequestId, BftMessage, LocalizedBftMessage, ConsensusVote, SignedConsensusVote, SignedConsensusMessage, SignedConsensusProposal, Status as StatusMessage};
 pub use error::Error;
 pub use config::{Roles, ProtocolConfig};
-pub use on_demand::{OnDemand, OnDemandService, RemoteCallResponse};
+pub use on_demand::{OnDemand, OnDemandService, RemoteResponse};

substrate/substrate/network/src/message.rs

@@ -18,7 +18,10 @@
 
 use runtime_primitives::traits::{Block as BlockT, Header as HeaderT};
 use codec::{Encode, Decode, Input, Output};
-pub use self::generic::{BlockAnnounce, RemoteCallRequest, ConsensusVote, SignedConsensusVote, FromBlock};
+pub use self::generic::{
+	BlockAnnounce, RemoteCallRequest, RemoteReadRequest,
+	ConsensusVote, SignedConsensusVote, FromBlock
+};
 
 /// A unique ID of a request.
 pub type RequestId = u64;
@@ -132,14 +135,25 @@ pub struct RemoteCallResponse {
 	pub proof: Vec<Vec<u8>>,
 }
 
+#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
+/// Remote read response.
+pub struct RemoteReadResponse {
+	/// Id of a request this response was made for.
+	pub id: RequestId,
+	/// Read proof.
+	pub proof: Vec<Vec<u8>>,
+}
+
 /// Generic types.
 pub mod generic {
 	use primitives::AuthorityId;
 	use runtime_primitives::bft::Justification;
 	use ed25519;
 	use service::Roles;
-	use super::{BlockAttributes, RemoteCallResponse, RequestId, Transactions, Direction};
-
+	use super::{
+		BlockAttributes, RemoteCallResponse, RemoteReadResponse,
+		RequestId, Transactions, Direction
+	};
 
 	/// Block data sent in the response.
 	#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
@@ -252,6 +266,10 @@ pub mod generic {
 		RemoteCallRequest(RemoteCallRequest<Hash>),
 		/// Remote method call response.
 		RemoteCallResponse(RemoteCallResponse),
+		/// Remote storage read request.
+		RemoteReadRequest(RemoteReadRequest<Hash>),
+		/// Remote storage read response.
+		RemoteReadResponse(RemoteReadResponse),
 		/// Chain-specific message
 		#[codec(index = "255")]
 		ChainSpecific(Vec<u8>),
@@ -319,4 +337,15 @@ pub mod generic {
 		/// Call data.
 		pub data: Vec<u8>,
 	}
+
+	#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
+	/// Remote storage read request.
+	pub struct RemoteReadRequest<H> {
+		/// Unique request id.
+		pub id: RequestId,
+		/// Block at which to perform call.
+		pub block: H,
+		/// Storage key.
+		pub key: Vec<u8>,
+	}
 }

substrate/substrate/network/src/on_demand.rs

@@ -25,7 +25,7 @@ use linked_hash_map::LinkedHashMap;
 use linked_hash_map::Entry;
 use parking_lot::Mutex;
 use client;
-use client::light::fetcher::{Fetcher, FetchChecker, RemoteCallRequest};
+use client::light::fetcher::{Fetcher, FetchChecker, RemoteCallRequest, RemoteReadRequest};
 use io::SyncIo;
 use message;
 use network_libp2p::{Severity, NodeIndex};
@@ -46,6 +46,9 @@ pub trait OnDemandService<Block: BlockT>: Send + Sync {
 	/// Maintain peers requests.
 	fn maintain_peers(&self, io: &mut SyncIo);
 
+	/// When read response is received from remote node.
+	fn on_remote_read_response(&self, io: &mut SyncIo, peer: NodeIndex, response: message::RemoteReadResponse);
+
 	/// When call response is received from remote node.
 	fn on_remote_call_response(&self, io: &mut SyncIo, peer: NodeIndex, response: message::RemoteCallResponse);
 }
@@ -57,8 +60,8 @@ pub struct OnDemand<B: BlockT, E: service::ExecuteInContext<B>> {
 }
 
 /// On-demand remote call response.
-pub struct RemoteCallResponse {
-	receiver: Receiver<Result<client::CallResult, client::error::Error>>,
+pub struct RemoteResponse<T> {
+	receiver: Receiver<Result<T, client::error::Error>>,
 }
 
 #[derive(Default)]
@@ -77,16 +80,18 @@ struct Request<Block: BlockT> {
 }
 
 enum RequestData<Block: BlockT> {
-	RemoteCall(RemoteCallRequest<Block::Hash>, Sender<Result<client::CallResult, client::error::Error>>),
+	RemoteRead(RemoteReadRequest<Block::Header>, Sender<Result<Option<Vec<u8>>, client::error::Error>>),
+	RemoteCall(RemoteCallRequest<Block::Header>, Sender<Result<client::CallResult, client::error::Error>>),
 }
 
 enum Accept<Block: BlockT> {
 	Ok,
 	CheckFailed(client::error::Error, RequestData<Block>),
+	Unexpected(RequestData<Block>),
 }
 
-impl Future for RemoteCallResponse {
-	type Item = client::CallResult;
+impl<T> Future for RemoteResponse<T> {
+	type Item = T;
 	type Error = client::error::Error;
 
 	fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
@@ -150,6 +155,10 @@ impl<B: BlockT, E> OnDemand<B, E> where
 				core.remove_peer(peer);
 				Some(retry_request_data)
 			},
+			Accept::Unexpected(retry_request_data) => {
+				trace!(target: "sync", "Unexpected response to remote {} from peer {}", rtype, peer);
+				Some(retry_request_data)
+			},
 		};
 
 		if let Some(request_data) = retry_request_data {
@@ -189,6 +198,20 @@ impl<B, E> OnDemandService<B> for OnDemand<B, E> where
 		core.dispatch();
 	}
 
+	fn on_remote_read_response(&self, io: &mut SyncIo, peer: NodeIndex, response: message::RemoteReadResponse) {
+		self.accept_response("read", io, peer, response.id, |request| match request.data {
+			RequestData::RemoteRead(request, sender) => match self.checker.check_read_proof(&request, response.proof) {
+				Ok(response) => {
+					// we do not bother if receiver has been dropped already
+					let _ = sender.send(Ok(response));
+					Accept::Ok
+				},
+				Err(error) => Accept::CheckFailed(error, RequestData::RemoteRead(request, sender)),
+			},
+			data @ _ => Accept::Unexpected(data),
+		})
+	}
+
 	fn on_remote_call_response(&self, io: &mut SyncIo, peer: NodeIndex, response: message::RemoteCallResponse) {
 		self.accept_response("call", io, peer, response.id, |request| match request.data {
 			RequestData::RemoteCall(request, sender) => match self.checker.check_execution_proof(&request, response.proof) {
@@ -199,6 +222,7 @@ impl<B, E> OnDemandService<B> for OnDemand<B, E> where
 				},
 				Err(error) => Accept::CheckFailed(error, RequestData::RemoteCall(request, sender)),
 			},
+			data @ _ => Accept::Unexpected(data),
 		})
 	}
 }
@@ -208,12 +232,19 @@ impl<B, E> Fetcher<B> for OnDemand<B, E> where
 	E: service::ExecuteInContext<B>,
 	B::Header: HeaderT,
 {
-	type RemoteCallResult = RemoteCallResponse;
+	type RemoteReadResult = RemoteResponse<Option<Vec<u8>>>;
+	type RemoteCallResult = RemoteResponse<client::CallResult>;
 
-	fn remote_call(&self, request: RemoteCallRequest<B::Hash>) -> Self::RemoteCallResult {
+	fn remote_read(&self, request: RemoteReadRequest<B::Header>) -> Self::RemoteReadResult {
+		let (sender, receiver) = channel();
+		self.schedule_request(RequestData::RemoteRead(request, sender),
+			RemoteResponse { receiver })
+	}
+
+	fn remote_call(&self, request: RemoteCallRequest<B::Header>) -> Self::RemoteCallResult {
 		let (sender, receiver) = channel();
 		self.schedule_request(RequestData::RemoteCall(request, sender),
-			RemoteCallResponse { receiver })
+			RemoteResponse { receiver })
 	}
 }
 
@@ -301,12 +332,19 @@ impl<B, E> OnDemandCore<B, E> where
 impl<Block: BlockT> Request<Block> {
 	pub fn message(&self) -> message::Message<Block> {
 		match self.data {
-			RequestData::RemoteCall(ref data, _) => message::generic::Message::RemoteCallRequest(message::RemoteCallRequest {
-				id: self.id,
-				block: data.block,
-				method: data.method.clone(),
-				data: data.call_data.clone(),
-			}),
+			RequestData::RemoteRead(ref data, _) => message::generic::Message::RemoteReadRequest(
+				message::RemoteReadRequest {
+					id: self.id,
+					block: data.block,
+					key: data.key.clone(),
+				}),
+			RequestData::RemoteCall(ref data, _) => message::generic::Message::RemoteCallRequest(
+				message::RemoteCallRequest {
+					id: self.id,
+					block: data.block,
+					method: data.method.clone(),
+					data: data.call_data.clone(),
+				}),
 		}
 	}
 }
@@ -319,13 +357,13 @@ pub mod tests {
 	use futures::Future;
 	use parking_lot::RwLock;
 	use client;
-	use client::light::fetcher::{Fetcher, FetchChecker, RemoteCallRequest};
+	use client::light::fetcher::{Fetcher, FetchChecker, RemoteCallRequest, RemoteReadRequest};
 	use message;
 	use network_libp2p::NodeIndex;
 	use service::{Roles, ExecuteInContext};
 	use test::TestIo;
 	use super::{REQUEST_TIMEOUT, OnDemand, OnDemandService};
-	use test_client::runtime::{Block, Hash};
+	use test_client::runtime::{Block, Header};
 
 	pub struct DummyExecutor;
 	struct DummyFetchChecker { ok: bool }
@@ -335,7 +373,14 @@ pub mod tests {
 	}
 
 	impl FetchChecker<Block> for DummyFetchChecker {
-		fn check_execution_proof(&self, _request: &RemoteCallRequest<Hash>, _remote_proof: Vec<Vec<u8>>) -> client::error::Result<client::CallResult> {
+		fn check_read_proof(&self, _request: &RemoteReadRequest<Header>, _remote_proof: Vec<Vec<u8>>) -> client::error::Result<Option<Vec<u8>>> {
+			match self.ok {
+				true => Ok(Some(vec![42])),
+				false => Err(client::error::ErrorKind::Backend("Test error".into()).into()),
+			}
+		}
+
+		fn check_execution_proof(&self, _request: &RemoteCallRequest<Header>, _remote_proof: Vec<Vec<u8>>) -> client::error::Result<client::CallResult> {
 			match self.ok {
 				true => Ok(client::CallResult {
 					return_data: vec![42],
@@ -365,6 +410,16 @@ pub mod tests {
 		});
 	}
 
+	fn dummy_header() -> Header {
+		Header {
+			parent_hash: Default::default(),
+			number: 0,
+			state_root: Default::default(),
+			extrinsics_root: Default::default(),
+			digest: Default::default(),
+		}
+	}
+
 	#[test]
 	fn knows_about_peers_roles() {
 		let (_, on_demand) = dummy(true);
@@ -394,7 +449,12 @@ pub mod tests {
 		assert_eq!(vec![0, 1], on_demand.core.lock().idle_peers.iter().cloned().collect::<Vec<_>>());
 		assert!(on_demand.core.lock().active_peers.is_empty());
 
-		on_demand.remote_call(RemoteCallRequest { block: Default::default(), method: "test".into(), call_data: vec![] });
+		on_demand.remote_call(RemoteCallRequest {
+			block: Default::default(),
+			header: dummy_header(),
+			method: "test".into(),
+			call_data: vec![],
+		});
 		assert_eq!(vec![1], on_demand.core.lock().idle_peers.iter().cloned().collect::<Vec<_>>());
 		assert_eq!(vec![0], on_demand.core.lock().active_peers.keys().cloned().collect::<Vec<_>>());
 
@@ -412,7 +472,12 @@ pub mod tests {
 		let mut network = TestIo::new(&queue, None);
 		on_demand.on_connect(0, Roles::FULL);
 
-		on_demand.remote_call(RemoteCallRequest { block: Default::default(), method: "test".into(), call_data: vec![] });
+		on_demand.remote_call(RemoteCallRequest {
+			block: Default::default(),
+			header: dummy_header(),
+			method: "test".into(),
+			call_data: vec![],
+		});
 		receive_call_response(&*on_demand, &mut network, 0, 1);
 		assert!(network.to_disconnect.contains(&0));
 		assert_eq!(on_demand.core.lock().pending_requests.len(), 1);
@@ -423,7 +488,12 @@ pub mod tests {
 		let (_x, on_demand) = dummy(false);
 		let queue = RwLock::new(VecDeque::new());
 		let mut network = TestIo::new(&queue, None);
-		on_demand.remote_call(RemoteCallRequest { block: Default::default(), method: "test".into(), call_data: vec![] });
+		on_demand.remote_call(RemoteCallRequest {
+			block: Default::default(),
+			header: dummy_header(),
+			method: "test".into(),
+			call_data: vec![],
+		});
 
 		on_demand.on_connect(0, Roles::FULL);
 		receive_call_response(&*on_demand, &mut network, 0, 0);
@@ -449,7 +519,12 @@ pub mod tests {
 		let mut network = TestIo::new(&queue, None);
 		on_demand.on_connect(0, Roles::FULL);
 
-		let response = on_demand.remote_call(RemoteCallRequest { block: Default::default(), method: "test".into(), call_data: vec![] });
+		let response = on_demand.remote_call(RemoteCallRequest {
+			block: Default::default(),
+			header: dummy_header(),
+			method: "test".into(),
+			call_data: vec![],
+		});
 		let thread = ::std::thread::spawn(move || {
 			let result = response.wait().unwrap();
 			assert_eq!(result.return_data, vec![42]);

substrate/substrate/network/src/protocol.rs

@@ -19,6 +19,7 @@ use std::{mem, cmp};
 use std::sync::Arc;
 use std::time;
 use parking_lot::RwLock;
+use rustc_hex::ToHex;
 use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Hash, HashFor, As};
 use runtime_primitives::generic::BlockId;
 use network_libp2p::{NodeIndex, Severity};
@@ -268,6 +269,8 @@ impl<B: BlockT, S: Specialization<B>> Protocol<B, S> {
 			GenericMessage::Transactions(m) => self.on_extrinsics(io, who, m),
 			GenericMessage::RemoteCallRequest(request) => self.on_remote_call_request(io, who, request),
 			GenericMessage::RemoteCallResponse(response) => self.on_remote_call_response(io, who, response),
+			GenericMessage::RemoteReadRequest(request) => self.on_remote_read_request(io, who, request),
+			GenericMessage::RemoteReadResponse(response) => self.on_remote_read_response(io, who, response),
 			other => self.specialization.write().on_message(&mut ProtocolContext::new(&self.context_data, io), who, other),
 		}
 	}
@@ -602,6 +605,26 @@ impl<B: BlockT, S: Specialization<B>> Protocol<B, S> {
 		self.on_demand.as_ref().map(|s| s.on_remote_call_response(io, who, response));
 	}
 
+	fn on_remote_read_request(&self, io: &mut SyncIo, who: NodeIndex, request: message::RemoteReadRequest<B::Hash>) {
+		trace!(target: "sync", "Remote read request {} from {} ({} at {})",
+			request.id, who, request.key.to_hex(), request.block);
+		let proof = match self.context_data.chain.read_proof(&request.block, &request.key) {
+			Ok(proof) => proof,
+			Err(error) => {
+				trace!(target: "sync", "Remote read request {} from {} ({} at {}) failed with: {}",
+					request.id, who, request.key.to_hex(), request.block, error);
+				Default::default()
+			},
+		};
+ 		self.send_message(io, who, GenericMessage::RemoteReadResponse(message::RemoteReadResponse {
+			id: request.id, proof,
+		}));
+	}
+ 	fn on_remote_read_response(&self, io: &mut SyncIo, who: NodeIndex, response: message::RemoteReadResponse) {
+		trace!(target: "sync", "Remote read response {} from {}", response.id, who);
+		self.on_demand.as_ref().map(|s| s.on_remote_read_response(io, who, response));
+	}
+
 	/// Execute a closure with access to a network context and specialization.
 	pub fn with_spec<F, U>(&self, io: &mut SyncIo, f: F) -> U
 		where F: FnOnce(&mut S, &mut Context<B>) -> U

substrate/substrate/service/src/components.rs

@@ -213,7 +213,10 @@ pub struct LightComponents<Factory: ServiceFactory> {
 	_factory: PhantomData<Factory>,
 }
 
-impl<Factory: ServiceFactory> Components for LightComponents<Factory> {
+impl<Factory: ServiceFactory> Components for LightComponents<Factory>
+	where
+		<<Factory as ServiceFactory>::Block as BlockT>::Hash: Into<[u8; 32]>,
+{
 	type Factory = Factory;
 	type Executor = LightExecutor<Factory>;
 	type Backend = LightBackend<Factory>;
@@ -236,7 +239,7 @@ impl<Factory: ServiceFactory> Components for LightComponents<Factory> {
 		};
 		let db_storage = client_db::light::LightStorage::new(db_settings)?;
 		let light_blockchain = client::light::new_light_blockchain(db_storage);
-		let fetch_checker = Arc::new(client::light::new_fetch_checker(light_blockchain.clone(), executor));
+		let fetch_checker = Arc::new(client::light::new_fetch_checker(executor));
 		let fetcher = Arc::new(network::OnDemand::new(fetch_checker));
 		let client_backend = client::light::new_light_backend(light_blockchain, fetcher.clone());
 		let client = client::light::new_light(client_backend, fetcher.clone(), &config.chain_spec)?;

substrate/substrate/state-machine/src/lib.rs

@@ -406,6 +406,30 @@ pub fn execution_proof_check<Exec: CodeExecutor>(
 	execute(&backend, overlay, exec, method, call_data, ExecutionStrategy::NativeWhenPossible)
 }
 
+/// Generate storage read proof.
+pub fn prove_read<B: TryIntoTrieBackend>(
+	backend: B,
+	key: &[u8],
+) -> Result<(Option<Vec<u8>>, Vec<Vec<u8>>), Box<Error>>
+{
+	let trie_backend = backend.try_into_trie_backend()
+		.ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box<Error>)?;
+	let proving_backend = proving_backend::ProvingBackend::new(trie_backend);
+	let result = proving_backend.storage(key).map_err(|e| Box::new(e) as Box<Error>)?;
+	Ok((result, proving_backend.extract_proof()))
+}
+
+/// Check storage read proof, generated by `prove_read` call.
+pub fn read_proof_check(
+	root: [u8; 32],
+	proof: Vec<Vec<u8>>,
+	key: &[u8],
+) -> Result<Option<Vec<u8>>, Box<Error>>
+{
+	let backend = proving_backend::create_proof_check_backend(root.into(), proof)?;
+	backend.storage(key).map_err(|e| Box::new(e) as Box<Error>)
+}
+
 #[cfg(test)]
 mod tests {
 	use super::*;
@@ -599,4 +623,18 @@ mod tests {
 			],
 		);
 	}
+
+	#[test]
+	fn prove_read_and_proof_check_works() {
+		// fetch read proof from 'remote' full node
+		let remote_backend = trie_backend::tests::test_trie();
+		let remote_root = remote_backend.storage_root(::std::iter::empty()).0;
+		let remote_proof = prove_read(remote_backend, b"value2").unwrap().1;
+ 		// check proof locally
+		let local_result1 = read_proof_check(remote_root, remote_proof.clone(), b"value2").unwrap();
+		let local_result2 = read_proof_check(remote_root, remote_proof.clone(), &[0xff]).is_ok();
+ 		// check that results are correct
+		assert_eq!(local_result1, Some(vec![24]));
+		assert_eq!(local_result2, false);
+	}
 }

substrate/substrate/state-machine/src/trie_backend.rs

@@ -282,6 +282,9 @@ pub mod tests {
 			trie.insert(b"value1", &[42]).unwrap();
 			trie.insert(b"value2", &[24]).unwrap();
 			trie.insert(b":code", b"return 42").unwrap();
+			for i in 128u8..255u8 {
+				trie.insert(&[i], &[i]).unwrap();
+			}
 		}
 		(mdb, root)
 	}