Run cargo fmt on the whole code base (#9394)

* Run cargo fmt on the whole code base

* Second run

* Add CI check

* Fix compilation

* More unnecessary braces

* Handle weights

* Use --all

* Use correct attributes...

* Fix UI tests

* AHHHHHHHHH

* 🤦

* Docs

* Fix compilation

* 🤷

* Please stop

* 🤦 x 2

* More

* make rustfmt.toml consistent with polkadot

Co-authored-by: André Silva <andrerfosilva@gmail.com>
This commit is contained in:
Bastian Köcher
2021-07-21 16:32:32 +02:00
committed by GitHub
parent d451c38c1c
commit 7b56ab15b4
1010 changed files with 53339 additions and 51208 deletions
+97 -89
View File
@@ -17,27 +17,33 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
config::ProtocolId,
bitswap::Bitswap,
config::ProtocolId,
discovery::{DiscoveryBehaviour, DiscoveryConfig, DiscoveryOut},
light_client_requests, peer_info,
protocol::{message::Roles, CustomMessageOutcome, NotificationsSink, Protocol},
peer_info, request_responses, light_client_requests,
ObservedRole, DhtEvent,
request_responses, DhtEvent, ObservedRole,
};
use bytes::Bytes;
use futures::{channel::oneshot, stream::StreamExt};
use libp2p::NetworkBehaviour;
use libp2p::core::{Multiaddr, PeerId, PublicKey};
use libp2p::identify::IdentifyInfo;
use libp2p::kad::record;
use libp2p::swarm::{
NetworkBehaviourAction, NetworkBehaviourEventProcess, PollParameters, toggle::Toggle
use libp2p::{
core::{Multiaddr, PeerId, PublicKey},
identify::IdentifyInfo,
kad::record,
swarm::{toggle::Toggle, NetworkBehaviourAction, NetworkBehaviourEventProcess, PollParameters},
NetworkBehaviour,
};
use log::debug;
use prost::Message;
use sp_consensus::{BlockOrigin, import_queue::{IncomingBlock, Origin}};
use sp_runtime::{traits::{Block as BlockT, NumberFor}, Justifications};
use sp_consensus::{
import_queue::{IncomingBlock, Origin},
BlockOrigin,
};
use sp_runtime::{
traits::{Block as BlockT, NumberFor},
Justifications,
};
use std::{
borrow::Cow,
collections::{HashSet, VecDeque},
@@ -47,8 +53,7 @@ use std::{
};
pub use crate::request_responses::{
ResponseFailure, InboundFailure, RequestFailure, OutboundFailure, RequestId,
IfDisconnected
IfDisconnected, InboundFailure, OutboundFailure, RequestFailure, RequestId, ResponseFailure,
};
/// General behaviour of the network. Combines all protocols together.
@@ -210,8 +215,9 @@ impl<B: BlockT> Behaviour<B> {
peer_info: peer_info::PeerInfoBehaviour::new(user_agent, local_public_key),
discovery: disco_config.finish(),
bitswap: bitswap.into(),
request_responses:
request_responses::RequestResponsesBehaviour::new(request_response_protocols.into_iter())?,
request_responses: request_responses::RequestResponsesBehaviour::new(
request_response_protocols.into_iter(),
)?,
light_client_request_sender,
events: VecDeque::new(),
block_request_protocol_name,
@@ -233,7 +239,9 @@ impl<B: BlockT> Behaviour<B> {
///
/// Identifies Kademlia instances by their [`ProtocolId`] and kbuckets by the base 2 logarithm
/// of their lower bound.
pub fn num_entries_per_kbucket(&mut self) -> impl ExactSizeIterator<Item = (&ProtocolId, Vec<(u32, usize)>)> {
pub fn num_entries_per_kbucket(
&mut self,
) -> impl ExactSizeIterator<Item = (&ProtocolId, Vec<(u32, usize)>)> {
self.discovery.num_entries_per_kbucket()
}
@@ -243,7 +251,9 @@ impl<B: BlockT> Behaviour<B> {
}
/// Returns the total size in bytes of all the records in the Kademlia record stores.
pub fn kademlia_records_total_size(&mut self) -> impl ExactSizeIterator<Item = (&ProtocolId, usize)> {
pub fn kademlia_records_total_size(
&mut self,
) -> impl ExactSizeIterator<Item = (&ProtocolId, usize)> {
self.discovery.kademlia_records_total_size()
}
@@ -265,7 +275,8 @@ impl<B: BlockT> Behaviour<B> {
pending_response: oneshot::Sender<Result<Vec<u8>, RequestFailure>>,
connect: IfDisconnected,
) {
self.request_responses.send_request(target, protocol, request, pending_response, connect)
self.request_responses
.send_request(target, protocol, request, pending_response, connect)
}
/// Returns a shared reference to the user protocol.
@@ -307,21 +318,20 @@ fn reported_roles_to_observed_role(roles: Roles) -> ObservedRole {
}
}
impl<B: BlockT> NetworkBehaviourEventProcess<void::Void> for
Behaviour<B> {
impl<B: BlockT> NetworkBehaviourEventProcess<void::Void> for Behaviour<B> {
fn inject_event(&mut self, event: void::Void) {
void::unreachable(event)
}
}
impl<B: BlockT> NetworkBehaviourEventProcess<CustomMessageOutcome<B>> for
Behaviour<B> {
impl<B: BlockT> NetworkBehaviourEventProcess<CustomMessageOutcome<B>> for Behaviour<B> {
fn inject_event(&mut self, event: CustomMessageOutcome<B>) {
match event {
CustomMessageOutcome::BlockImport(origin, blocks) =>
self.events.push_back(BehaviourOut::BlockImport(origin, blocks)),
CustomMessageOutcome::JustificationImport(origin, hash, nb, justification) =>
self.events.push_back(BehaviourOut::JustificationImport(origin, hash, nb, justification)),
CustomMessageOutcome::JustificationImport(origin, hash, nb, justification) => self
.events
.push_back(BehaviourOut::JustificationImport(origin, hash, nb, justification)),
CustomMessageOutcome::BlockRequest { target, request, pending_response } => {
let mut buf = Vec::with_capacity(request.encoded_len());
if let Err(err) = request.encode(&mut buf) {
@@ -334,7 +344,11 @@ Behaviour<B> {
}
self.request_responses.send_request(
&target, &self.block_request_protocol_name, buf, pending_response, IfDisconnected::ImmediateError,
&target,
&self.block_request_protocol_name,
buf,
pending_response,
IfDisconnected::ImmediateError,
);
},
CustomMessageOutcome::StateRequest { target, request, pending_response } => {
@@ -349,11 +363,19 @@ Behaviour<B> {
}
self.request_responses.send_request(
&target, &self.state_request_protocol_name, buf, pending_response, IfDisconnected::ImmediateError,
&target,
&self.state_request_protocol_name,
buf,
pending_response,
IfDisconnected::ImmediateError,
);
},
CustomMessageOutcome::NotificationStreamOpened {
remote, protocol, negotiated_fallback, roles, notifications_sink
remote,
protocol,
negotiated_fallback,
roles,
notifications_sink,
} => {
self.events.push_back(BehaviourOut::NotificationStreamOpened {
remote,
@@ -363,32 +385,33 @@ Behaviour<B> {
notifications_sink: notifications_sink.clone(),
});
},
CustomMessageOutcome::NotificationStreamReplaced { remote, protocol, notifications_sink } =>
self.events.push_back(BehaviourOut::NotificationStreamReplaced {
remote,
protocol,
notifications_sink,
}),
CustomMessageOutcome::NotificationStreamClosed { remote, protocol } =>
self.events.push_back(BehaviourOut::NotificationStreamClosed {
remote,
protocol,
}),
CustomMessageOutcome::NotificationStreamReplaced {
remote,
protocol,
notifications_sink,
} => self.events.push_back(BehaviourOut::NotificationStreamReplaced {
remote,
protocol,
notifications_sink,
}),
CustomMessageOutcome::NotificationStreamClosed { remote, protocol } => self
.events
.push_back(BehaviourOut::NotificationStreamClosed { remote, protocol }),
CustomMessageOutcome::NotificationsReceived { remote, messages } => {
self.events.push_back(BehaviourOut::NotificationsReceived { remote, messages });
},
CustomMessageOutcome::PeerNewBest(peer_id, number) => {
self.light_client_request_sender.update_best_block(&peer_id, number);
}
},
CustomMessageOutcome::SyncConnected(peer_id) => {
self.light_client_request_sender.inject_connected(peer_id);
self.events.push_back(BehaviourOut::SyncConnected(peer_id))
}
},
CustomMessageOutcome::SyncDisconnected(peer_id) => {
self.light_client_request_sender.inject_disconnected(peer_id);
self.events.push_back(BehaviourOut::SyncDisconnected(peer_id))
}
CustomMessageOutcome::None => {}
},
CustomMessageOutcome::None => {},
}
}
}
@@ -397,38 +420,29 @@ impl<B: BlockT> NetworkBehaviourEventProcess<request_responses::Event> for Behav
fn inject_event(&mut self, event: request_responses::Event) {
match event {
request_responses::Event::InboundRequest { peer, protocol, result } => {
self.events.push_back(BehaviourOut::InboundRequest {
peer,
protocol,
result,
});
}
self.events.push_back(BehaviourOut::InboundRequest { peer, protocol, result });
},
request_responses::Event::RequestFinished { peer, protocol, duration, result } => {
self.events.push_back(BehaviourOut::RequestFinished {
peer, protocol, duration, result,
peer,
protocol,
duration,
result,
});
},
request_responses::Event::ReputationChanges { peer, changes } => {
request_responses::Event::ReputationChanges { peer, changes } =>
for change in changes {
self.substrate.report_peer(peer, change);
}
}
},
}
}
}
impl<B: BlockT> NetworkBehaviourEventProcess<peer_info::PeerInfoEvent>
for Behaviour<B> {
impl<B: BlockT> NetworkBehaviourEventProcess<peer_info::PeerInfoEvent> for Behaviour<B> {
fn inject_event(&mut self, event: peer_info::PeerInfoEvent) {
let peer_info::PeerInfoEvent::Identified {
peer_id,
info: IdentifyInfo {
protocol_version,
agent_version,
mut listen_addrs,
protocols,
..
},
info: IdentifyInfo { protocol_version, agent_version, mut listen_addrs, protocols, .. },
} = event;
if listen_addrs.len() > 30 {
@@ -447,8 +461,7 @@ impl<B: BlockT> NetworkBehaviourEventProcess<peer_info::PeerInfoEvent>
}
}
impl<B: BlockT> NetworkBehaviourEventProcess<DiscoveryOut>
for Behaviour<B> {
impl<B: BlockT> NetworkBehaviourEventProcess<DiscoveryOut> for Behaviour<B> {
fn inject_event(&mut self, out: DiscoveryOut) {
match out {
DiscoveryOut::UnroutablePeer(_peer_id) => {
@@ -456,27 +469,28 @@ impl<B: BlockT> NetworkBehaviourEventProcess<DiscoveryOut>
// to Kademlia is handled by the `Identify` protocol, part of the
// `PeerInfoBehaviour`. See the `NetworkBehaviourEventProcess`
// implementation for `PeerInfoEvent`.
}
},
DiscoveryOut::Discovered(peer_id) => {
self.substrate.add_default_set_discovered_nodes(iter::once(peer_id));
}
},
DiscoveryOut::ValueFound(results, duration) => {
self.events.push_back(BehaviourOut::Dht(DhtEvent::ValueFound(results), duration));
}
self.events
.push_back(BehaviourOut::Dht(DhtEvent::ValueFound(results), duration));
},
DiscoveryOut::ValueNotFound(key, duration) => {
self.events.push_back(BehaviourOut::Dht(DhtEvent::ValueNotFound(key), duration));
}
},
DiscoveryOut::ValuePut(key, duration) => {
self.events.push_back(BehaviourOut::Dht(DhtEvent::ValuePut(key), duration));
}
},
DiscoveryOut::ValuePutFailed(key, duration) => {
self.events.push_back(BehaviourOut::Dht(DhtEvent::ValuePutFailed(key), duration));
}
DiscoveryOut::RandomKademliaStarted(protocols) => {
self.events
.push_back(BehaviourOut::Dht(DhtEvent::ValuePutFailed(key), duration));
},
DiscoveryOut::RandomKademliaStarted(protocols) =>
for protocol in protocols {
self.events.push_back(BehaviourOut::RandomKademliaStarted(protocol));
}
}
},
}
}
}
@@ -488,22 +502,16 @@ impl<B: BlockT> Behaviour<B> {
_: &mut impl PollParameters,
) -> Poll<NetworkBehaviourAction<TEv, BehaviourOut<B>>> {
use light_client_requests::sender::OutEvent;
while let Poll::Ready(Some(event)) =
self.light_client_request_sender.poll_next_unpin(cx)
{
while let Poll::Ready(Some(event)) = self.light_client_request_sender.poll_next_unpin(cx) {
match event {
OutEvent::SendRequest {
target,
request,
pending_response,
protocol_name,
} => self.request_responses.send_request(
&target,
&protocol_name,
request,
pending_response,
IfDisconnected::ImmediateError,
),
OutEvent::SendRequest { target, request, pending_response, protocol_name } =>
self.request_responses.send_request(
&target,
&protocol_name,
request,
pending_response,
IfDisconnected::ImmediateError,
),
}
}
+46 -48
View File
@@ -20,31 +20,39 @@
//! Only supports bitswap 1.2.0.
//! CID is expected to reference 256-bit Blake2b transaction hash.
use std::collections::VecDeque;
use std::io;
use std::sync::Arc;
use std::task::{Context, Poll};
use crate::{
chain::Client,
schema::bitswap::{
message::{wantlist::WantType, Block as MessageBlock, BlockPresence, BlockPresenceType},
Message as BitswapMessage,
},
};
use cid::Version;
use core::pin::Pin;
use futures::Future;
use futures::io::{AsyncRead, AsyncWrite};
use libp2p::core::{
connection::ConnectionId, Multiaddr, PeerId,
upgrade, InboundUpgrade, OutboundUpgrade, UpgradeInfo,
use futures::{
io::{AsyncRead, AsyncWrite},
Future,
};
use libp2p::swarm::{
NetworkBehaviour, NetworkBehaviourAction, NotifyHandler, PollParameters,
ProtocolsHandler, IntoProtocolsHandler, OneShotHandler,
use libp2p::{
core::{
connection::ConnectionId, upgrade, InboundUpgrade, Multiaddr, OutboundUpgrade, PeerId,
UpgradeInfo,
},
swarm::{
IntoProtocolsHandler, NetworkBehaviour, NetworkBehaviourAction, NotifyHandler,
OneShotHandler, PollParameters, ProtocolsHandler,
},
};
use log::{error, debug, trace};
use log::{debug, error, trace};
use prost::Message;
use sp_runtime::traits::{Block as BlockT};
use unsigned_varint::{encode as varint_encode};
use crate::chain::Client;
use crate::schema::bitswap::{
Message as BitswapMessage,
message::{wantlist::WantType, Block as MessageBlock, BlockPresenceType, BlockPresence},
use sp_runtime::traits::Block as BlockT;
use std::{
collections::VecDeque,
io,
sync::Arc,
task::{Context, Poll},
};
use unsigned_varint::encode as varint_encode;
const LOG_TARGET: &str = "bitswap";
@@ -182,10 +190,7 @@ pub struct Bitswap<B> {
impl<B: BlockT> Bitswap<B> {
/// Create a new instance of the bitswap protocol handler.
pub fn new(client: Arc<dyn Client<B>>) -> Self {
Bitswap {
client,
ready_blocks: Default::default(),
}
Bitswap { client, ready_blocks: Default::default() }
}
}
@@ -201,11 +206,9 @@ impl<B: BlockT> NetworkBehaviour for Bitswap<B> {
Vec::new()
}
fn inject_connected(&mut self, _peer: &PeerId) {
}
fn inject_connected(&mut self, _peer: &PeerId) {}
fn inject_disconnected(&mut self, _peer: &PeerId) {
}
fn inject_disconnected(&mut self, _peer: &PeerId) {}
fn inject_event(&mut self, peer: PeerId, _connection: ConnectionId, message: HandlerEvent) {
let request = match message {
@@ -215,7 +218,7 @@ impl<B: BlockT> NetworkBehaviour for Bitswap<B> {
trace!(target: LOG_TARGET, "Received request: {:?} from {}", request, peer);
if self.ready_blocks.len() > MAX_RESPONSE_QUEUE {
debug!(target: LOG_TARGET, "Ignored request: queue is full");
return;
return
}
let mut response = BitswapMessage {
wantlist: None,
@@ -227,29 +230,25 @@ impl<B: BlockT> NetworkBehaviour for Bitswap<B> {
let wantlist = match request.wantlist {
Some(wantlist) => wantlist,
None => {
debug!(
target: LOG_TARGET,
"Unexpected bitswap message from {}",
peer,
);
return;
}
debug!(target: LOG_TARGET, "Unexpected bitswap message from {}", peer,);
return
},
};
if wantlist.entries.len() > MAX_WANTED_BLOCKS {
trace!(target: LOG_TARGET, "Ignored request: too many entries");
return;
return
}
for entry in wantlist.entries {
let cid = match cid::Cid::read_bytes(entry.block.as_slice()) {
Ok(cid) => cid,
Err(e) => {
trace!(target: LOG_TARGET, "Bad CID {:?}: {:?}", entry.block, e);
continue;
}
continue
},
};
if cid.version() != cid::Version::V1
|| cid.hash().code() != u64::from(cid::multihash::Code::Blake2b256)
|| cid.hash().size() != 32
if cid.version() != cid::Version::V1 ||
cid.hash().code() != u64::from(cid::multihash::Code::Blake2b256) ||
cid.hash().size() != 32
{
debug!(target: LOG_TARGET, "Ignoring unsupported CID {}: {}", peer, cid);
continue
@@ -261,7 +260,7 @@ impl<B: BlockT> NetworkBehaviour for Bitswap<B> {
Err(e) => {
error!(target: LOG_TARGET, "Error retrieving transaction {}: {}", hash, e);
None
}
},
};
match transaction {
Some(transaction) => {
@@ -273,10 +272,9 @@ impl<B: BlockT> NetworkBehaviour for Bitswap<B> {
mh_type: cid.hash().code(),
mh_len: cid.hash().size(),
};
response.payload.push(MessageBlock {
prefix: prefix.to_bytes(),
data: transaction,
});
response
.payload
.push(MessageBlock { prefix: prefix.to_bytes(), data: transaction });
} else {
response.block_presences.push(BlockPresence {
r#type: BlockPresenceType::Have as i32,
@@ -292,7 +290,7 @@ impl<B: BlockT> NetworkBehaviour for Bitswap<B> {
cid: cid.to_bytes(),
});
}
}
},
}
}
trace!(target: LOG_TARGET, "Response: {:?}", response);
@@ -304,7 +302,7 @@ impl<B: BlockT> NetworkBehaviour for Bitswap<B> {
<<Self::ProtocolsHandler as IntoProtocolsHandler>::Handler as ProtocolsHandler>::InEvent,
Self::OutEvent,
>,
> {
>{
if let Some((peer_id, message)) = self.ready_blocks.pop_front() {
return Poll::Ready(NetworkBehaviourAction::NotifyHandler {
peer_id: peer_id.clone(),
@@ -17,25 +17,32 @@
//! Helper for handling (i.e. answering) block requests from a remote peer via the
//! [`crate::request_responses::RequestResponsesBehaviour`].
use codec::{Encode, Decode};
use crate::chain::Client;
use crate::config::ProtocolId;
use crate::protocol::{message::BlockAttributes};
use crate::request_responses::{IncomingRequest, OutgoingResponse, ProtocolConfig};
use crate::schema::v1::block_request::FromBlock;
use crate::schema::v1::{BlockResponse, Direction};
use crate::{PeerId, ReputationChange};
use futures::channel::{mpsc, oneshot};
use futures::stream::StreamExt;
use crate::{
chain::Client,
config::ProtocolId,
protocol::message::BlockAttributes,
request_responses::{IncomingRequest, OutgoingResponse, ProtocolConfig},
schema::v1::{block_request::FromBlock, BlockResponse, Direction},
PeerId, ReputationChange,
};
use codec::{Decode, Encode};
use futures::{
channel::{mpsc, oneshot},
stream::StreamExt,
};
use log::debug;
use lru::LruCache;
use prost::Message;
use sp_runtime::generic::BlockId;
use sp_runtime::traits::{Block as BlockT, Header, One, Zero};
use std::cmp::min;
use std::sync::Arc;
use std::time::Duration;
use std::hash::{Hasher, Hash};
use sp_runtime::{
generic::BlockId,
traits::{Block as BlockT, Header, One, Zero},
};
use std::{
cmp::min,
hash::{Hash, Hasher},
sync::Arc,
time::Duration,
};
const LOG_TARGET: &str = "sync";
const MAX_BLOCKS_IN_RESPONSE: usize = 128;
@@ -61,7 +68,6 @@ pub fn generate_protocol_config(protocol_id: &ProtocolId) -> ProtocolConfig {
}
/// Generate the block protocol name from chain specific protocol identifier.
//
// Visibility `pub(crate)` to allow `crate::light_client_requests::sender` to generate block request
// protocol name and send block requests.
pub(crate) fn generate_protocol_name(protocol_id: &ProtocolId) -> String {
@@ -139,9 +145,7 @@ impl<B: BlockT> BlockRequestHandler<B> {
Ok(()) => debug!(target: LOG_TARGET, "Handled block request from {}.", peer),
Err(e) => debug!(
target: LOG_TARGET,
"Failed to handle block request from {}: {}",
peer,
e,
"Failed to handle block request from {}: {}", peer, e,
),
}
}
@@ -159,11 +163,11 @@ impl<B: BlockT> BlockRequestHandler<B> {
FromBlock::Hash(ref h) => {
let h = Decode::decode(&mut h.as_ref())?;
BlockId::<B>::Hash(h)
}
},
FromBlock::Number(ref n) => {
let n = Decode::decode(&mut n.as_ref())?;
BlockId::<B>::Number(n)
}
},
};
let max_blocks = if request.max_blocks == 0 {
@@ -172,8 +176,8 @@ impl<B: BlockT> BlockRequestHandler<B> {
min(request.max_blocks as usize, MAX_BLOCKS_IN_RESPONSE)
};
let direction = Direction::from_i32(request.direction)
.ok_or(HandleRequestError::ParseDirection)?;
let direction =
Direction::from_i32(request.direction).ok_or(HandleRequestError::ParseDirection)?;
let attributes = BlockAttributes::from_be_u32(request.fields)?;
@@ -201,7 +205,7 @@ impl<B: BlockT> BlockRequestHandler<B> {
},
None => {
self.seen_requests.put(key.clone(), SeenRequestsValue::First);
}
},
}
debug!(
@@ -247,11 +251,13 @@ impl<B: BlockT> BlockRequestHandler<B> {
Err(())
};
pending_response.send(OutgoingResponse {
result,
reputation_changes: reputation_change.into_iter().collect(),
sent_feedback: None,
}).map_err(|_| HandleRequestError::SendResponse)
pending_response
.send(OutgoingResponse {
result,
reputation_changes: reputation_change.into_iter().collect(),
sent_feedback: None,
})
.map_err(|_| HandleRequestError::SendResponse)
}
fn get_block_response(
@@ -298,10 +304,8 @@ impl<B: BlockT> BlockRequestHandler<B> {
let justification =
justifications.and_then(|just| just.into_justification(*b"FRNK"));
let is_empty_justification = justification
.as_ref()
.map(|j| j.is_empty())
.unwrap_or(false);
let is_empty_justification =
justification.as_ref().map(|j| j.is_empty()).unwrap_or(false);
let justification = justification.unwrap_or_default();
@@ -310,25 +314,27 @@ impl<B: BlockT> BlockRequestHandler<B> {
let body = if get_body {
match self.client.block_body(&BlockId::Hash(hash))? {
Some(mut extrinsics) => extrinsics.iter_mut()
.map(|extrinsic| extrinsic.encode())
.collect(),
Some(mut extrinsics) =>
extrinsics.iter_mut().map(|extrinsic| extrinsic.encode()).collect(),
None => {
log::trace!(target: LOG_TARGET, "Missing data for block request.");
break;
}
break
},
}
} else {
Vec::new()
};
let indexed_body = if get_indexed_body {
let indexed_body = if get_indexed_body {
match self.client.block_indexed_body(&BlockId::Hash(hash))? {
Some(transactions) => transactions,
None => {
log::trace!(target: LOG_TARGET, "Missing indexed block data for block request.");
break;
}
log::trace!(
target: LOG_TARGET,
"Missing indexed block data for block request."
);
break
},
}
} else {
Vec::new()
@@ -336,11 +342,7 @@ impl<B: BlockT> BlockRequestHandler<B> {
let block_data = crate::schema::v1::BlockData {
hash: hash.encode(),
header: if get_header {
header.encode()
} else {
Vec::new()
},
header: if get_header { header.encode() } else { Vec::new() },
body,
receipt: Vec::new(),
message_queue: Vec::new(),
@@ -358,15 +360,13 @@ impl<B: BlockT> BlockRequestHandler<B> {
}
match direction {
Direction::Ascending => {
block_id = BlockId::Number(number + One::one())
}
Direction::Ascending => block_id = BlockId::Number(number + One::one()),
Direction::Descending => {
if number.is_zero() {
break
}
block_id = BlockId::Hash(parent_hash)
}
},
}
}
+22 -10
View File
@@ -18,18 +18,30 @@
//! Blockchain access trait
use sp_blockchain::{Error, HeaderBackend, HeaderMetadata};
use sc_client_api::{BlockBackend, ProofProvider};
pub use sc_client_api::{ImportedState, StorageData, StorageKey};
use sp_blockchain::{Error, HeaderBackend, HeaderMetadata};
use sp_runtime::traits::{Block as BlockT, BlockIdTo};
pub use sc_client_api::{StorageKey, StorageData, ImportedState};
/// Local client abstraction for the network.
pub trait Client<Block: BlockT>: HeaderBackend<Block> + ProofProvider<Block> + BlockIdTo<Block, Error = Error>
+ BlockBackend<Block> + HeaderMetadata<Block, Error = Error> + Send + Sync
{}
pub trait Client<Block: BlockT>:
HeaderBackend<Block>
+ ProofProvider<Block>
+ BlockIdTo<Block, Error = Error>
+ BlockBackend<Block>
+ HeaderMetadata<Block, Error = Error>
+ Send
+ Sync
{
}
impl<Block: BlockT, T> Client<Block> for T
where
T: HeaderBackend<Block> + ProofProvider<Block> + BlockIdTo<Block, Error = Error>
+ BlockBackend<Block> + HeaderMetadata<Block, Error = Error> + Send + Sync
{}
impl<Block: BlockT, T> Client<Block> for T where
T: HeaderBackend<Block>
+ ProofProvider<Block>
+ BlockIdTo<Block, Error = Error>
+ BlockBackend<Block>
+ HeaderMetadata<Block, Error = Error>
+ Send
+ Sync
{
}
+62 -91
View File
@@ -21,14 +21,14 @@
//! The [`Params`] struct is the struct that must be passed in order to initialize the networking.
//! See the documentation of [`Params`].
pub use crate::chain::Client;
pub use crate::on_demand_layer::{AlwaysBadChecker, OnDemand};
pub use crate::request_responses::{
IncomingRequest,
OutgoingResponse,
ProtocolConfig as RequestResponseConfig,
pub use crate::{
chain::Client,
on_demand_layer::{AlwaysBadChecker, OnDemand},
request_responses::{
IncomingRequest, OutgoingResponse, ProtocolConfig as RequestResponseConfig,
},
};
pub use libp2p::{identity, core::PublicKey, wasm_ext::ExtTransport, build_multiaddr};
pub use libp2p::{build_multiaddr, core::PublicKey, identity, wasm_ext::ExtTransport};
// Note: this re-export shouldn't be part of the public API of the crate and will be removed in
// the future.
@@ -46,15 +46,19 @@ use libp2p::{
use prometheus_endpoint::Registry;
use sp_consensus::{block_validation::BlockAnnounceValidator, import_queue::ImportQueue};
use sp_runtime::traits::Block as BlockT;
use std::{borrow::Cow, convert::TryFrom, future::Future, pin::Pin, str::FromStr};
use std::{
borrow::Cow,
collections::HashMap,
convert::TryFrom,
error::Error,
fs,
future::Future,
io::{self, Write},
net::Ipv4Addr,
path::{Path, PathBuf},
pin::Pin,
str,
str::FromStr,
sync::Arc,
};
use zeroize::Zeroize;
@@ -181,7 +185,7 @@ pub enum TransactionImport {
}
/// Future resolving to transaction import result.
pub type TransactionImportFuture = Pin<Box<dyn Future<Output=TransactionImport> + Send>>;
pub type TransactionImportFuture = Pin<Box<dyn Future<Output = TransactionImport> + Send>>;
/// Transaction pool interface
pub trait TransactionPool<H: ExHashT, B: BlockT>: Send + Sync {
@@ -192,10 +196,7 @@ pub trait TransactionPool<H: ExHashT, B: BlockT>: Send + Sync {
/// Import a transaction into the pool.
///
/// This will return future.
fn import(
&self,
transaction: B::Extrinsic,
) -> TransactionImportFuture;
fn import(&self, transaction: B::Extrinsic) -> TransactionImportFuture;
/// Notify the pool about transactions broadcast.
fn on_broadcasted(&self, propagations: HashMap<H, Vec<String>>);
/// Get transaction by hash.
@@ -219,16 +220,15 @@ impl<H: ExHashT + Default, B: BlockT> TransactionPool<H, B> for EmptyTransaction
Default::default()
}
fn import(
&self,
_transaction: B::Extrinsic
) -> TransactionImportFuture {
fn import(&self, _transaction: B::Extrinsic) -> TransactionImportFuture {
Box::pin(future::ready(TransactionImport::KnownGood))
}
fn on_broadcasted(&self, _: HashMap<H, Vec<String>>) {}
fn transaction(&self, _h: &H) -> Option<B::Extrinsic> { None }
fn transaction(&self, _h: &H) -> Option<B::Extrinsic> {
None
}
}
/// Name of a protocol, transmitted on the wire. Should be unique for each chain. Always UTF-8.
@@ -267,17 +267,16 @@ impl fmt::Debug for ProtocolId {
/// assert_eq!(peer_id, "QmSk5HQbn6LhUwDiNMseVUjuRYhEtYj4aUZ6WfWoGURpdV".parse::<PeerId>().unwrap());
/// assert_eq!(addr, "/ip4/198.51.100.19/tcp/30333".parse::<Multiaddr>().unwrap());
/// ```
///
pub fn parse_str_addr(addr_str: &str) -> Result<(PeerId, Multiaddr), ParseErr> {
let addr: Multiaddr = addr_str.parse()?;
parse_addr(addr)
}
/// Splits a Multiaddress into a Multiaddress and PeerId.
pub fn parse_addr(mut addr: Multiaddr)-> Result<(PeerId, Multiaddr), ParseErr> {
pub fn parse_addr(mut addr: Multiaddr) -> Result<(PeerId, Multiaddr), ParseErr> {
let who = match addr.pop() {
Some(multiaddr::Protocol::P2p(key)) => PeerId::from_multihash(key)
.map_err(|_| ParseErr::InvalidPeerId)?,
Some(multiaddr::Protocol::P2p(key)) =>
PeerId::from_multihash(key).map_err(|_| ParseErr::InvalidPeerId)?,
_ => return Err(ParseErr::PeerIdMissing),
};
@@ -325,10 +324,7 @@ impl FromStr for MultiaddrWithPeerId {
fn from_str(s: &str) -> Result<Self, Self::Err> {
let (peer_id, multiaddr) = parse_str_addr(s)?;
Ok(MultiaddrWithPeerId {
peer_id,
multiaddr,
})
Ok(MultiaddrWithPeerId { peer_id, multiaddr })
}
}
@@ -504,18 +500,13 @@ impl NetworkConfiguration {
/// Create new default configuration for localhost-only connection with random port (useful for testing)
pub fn new_local() -> NetworkConfiguration {
let mut config = NetworkConfiguration::new(
"test-node",
"test-client",
Default::default(),
None,
);
let mut config =
NetworkConfiguration::new("test-node", "test-client", Default::default(), None);
config.listen_addresses = vec![
iter::once(multiaddr::Protocol::Ip4(Ipv4Addr::new(127, 0, 0, 1)))
config.listen_addresses =
vec![iter::once(multiaddr::Protocol::Ip4(Ipv4Addr::new(127, 0, 0, 1)))
.chain(iter::once(multiaddr::Protocol::Tcp(0)))
.collect()
];
.collect()];
config.allow_non_globals_in_dht = true;
config
@@ -523,18 +514,13 @@ impl NetworkConfiguration {
/// Create new default configuration for localhost-only connection with random port (useful for testing)
pub fn new_memory() -> NetworkConfiguration {
let mut config = NetworkConfiguration::new(
"test-node",
"test-client",
Default::default(),
None,
);
let mut config =
NetworkConfiguration::new("test-node", "test-client", Default::default(), None);
config.listen_addresses = vec![
iter::once(multiaddr::Protocol::Ip4(Ipv4Addr::new(127, 0, 0, 1)))
config.listen_addresses =
vec![iter::once(multiaddr::Protocol::Ip4(Ipv4Addr::new(127, 0, 0, 1)))
.chain(iter::once(multiaddr::Protocol::Tcp(0)))
.collect()
];
.collect()];
config.allow_non_globals_in_dht = true;
config
@@ -674,7 +660,7 @@ impl NonReservedPeerMode {
#[derive(Clone, Debug)]
pub enum NodeKeyConfig {
/// A Ed25519 secret key configuration.
Ed25519(Secret<ed25519::SecretKey>)
Ed25519(Secret<ed25519::SecretKey>),
}
impl Default for NodeKeyConfig {
@@ -698,7 +684,7 @@ pub enum Secret<K> {
/// * `ed25519::SecretKey`: An unencoded 32 bytes Ed25519 secret key.
File(PathBuf),
/// Always generate a new secret key `K`.
New
New,
}
impl<K> fmt::Debug for Secret<K> {
@@ -725,35 +711,27 @@ impl NodeKeyConfig {
pub fn into_keypair(self) -> io::Result<Keypair> {
use NodeKeyConfig::*;
match self {
Ed25519(Secret::New) =>
Ok(Keypair::generate_ed25519()),
Ed25519(Secret::New) => Ok(Keypair::generate_ed25519()),
Ed25519(Secret::Input(k)) =>
Ok(Keypair::Ed25519(k.into())),
Ed25519(Secret::Input(k)) => Ok(Keypair::Ed25519(k.into())),
Ed25519(Secret::File(f)) =>
get_secret(
f,
|mut b| {
match String::from_utf8(b.to_vec())
.ok()
.and_then(|s|{
if s.len() == 64 {
hex::decode(&s).ok()
} else {
None
}}
)
{
Some(s) => ed25519::SecretKey::from_bytes(s),
_ => ed25519::SecretKey::from_bytes(&mut b),
}
},
ed25519::SecretKey::generate,
|b| b.as_ref().to_vec()
)
.map(ed25519::Keypair::from)
.map(Keypair::Ed25519),
Ed25519(Secret::File(f)) => get_secret(
f,
|mut b| match String::from_utf8(b.to_vec()).ok().and_then(|s| {
if s.len() == 64 {
hex::decode(&s).ok()
} else {
None
}
}) {
Some(s) => ed25519::SecretKey::from_bytes(s),
_ => ed25519::SecretKey::from_bytes(&mut b),
},
ed25519::SecretKey::generate,
|b| b.as_ref().to_vec(),
)
.map(ed25519::Keypair::from)
.map(Keypair::Ed25519),
}
}
}
@@ -770,9 +748,9 @@ where
W: Fn(&K) -> Vec<u8>,
{
std::fs::read(&file)
.and_then(|mut sk_bytes|
parse(&mut sk_bytes)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)))
.and_then(|mut sk_bytes| {
parse(&mut sk_bytes).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
})
.or_else(|e| {
if e.kind() == io::ErrorKind::NotFound {
file.as_ref().parent().map_or(Ok(()), fs::create_dir_all)?;
@@ -790,7 +768,7 @@ where
/// Write secret bytes to a file.
fn write_secret_file<P>(path: P, sk_bytes: &[u8]) -> io::Result<()>
where
P: AsRef<Path>
P: AsRef<Path>,
{
let mut file = open_secret_file(&path)?;
file.write_all(sk_bytes)
@@ -800,26 +778,19 @@ where
#[cfg(unix)]
fn open_secret_file<P>(path: P) -> io::Result<fs::File>
where
P: AsRef<Path>
P: AsRef<Path>,
{
use std::os::unix::fs::OpenOptionsExt;
fs::OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(path)
fs::OpenOptions::new().write(true).create_new(true).mode(0o600).open(path)
}
/// Opens a file containing a secret key in write mode.
#[cfg(not(unix))]
fn open_secret_file<P>(path: P) -> Result<fs::File, io::Error>
where
P: AsRef<Path>
P: AsRef<Path>,
{
fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(path)
fs::OpenOptions::new().write(true).create_new(true).open(path)
}
#[cfg(test)]
@@ -835,7 +806,7 @@ mod tests {
match kp {
Keypair::Ed25519(p) => p.secret().as_ref().iter().cloned().collect(),
Keypair::Secp256k1(p) => p.secret().to_bytes().to_vec(),
_ => panic!("Unexpected keypair.")
_ => panic!("Unexpected keypair."),
}
}
+262 -175
View File
@@ -45,28 +45,43 @@
//! **Important**: In order for the discovery mechanism to work properly, there needs to be an
//! active mechanism that asks nodes for the addresses they are listening on. Whenever we learn
//! of a node's address, you must call `add_self_reported_address`.
//!
use crate::config::ProtocolId;
use crate::utils::LruHashSet;
use crate::{config::ProtocolId, utils::LruHashSet};
use futures::prelude::*;
use futures_timer::Delay;
use ip_network::IpNetwork;
use libp2p::core::{connection::{ConnectionId, ListenerId}, ConnectedPoint, Multiaddr, PeerId, PublicKey};
use libp2p::swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters, ProtocolsHandler, IntoProtocolsHandler};
use libp2p::swarm::protocols_handler::multi::IntoMultiHandler;
use libp2p::kad::{Kademlia, KademliaBucketInserts, KademliaConfig, KademliaEvent, QueryResult, Quorum, Record};
use libp2p::kad::GetClosestPeersError;
use libp2p::kad::handler::KademliaHandlerProto;
use libp2p::kad::QueryId;
use libp2p::kad::record::{self, store::{MemoryStore, RecordStore}};
#[cfg(not(target_os = "unknown"))]
use libp2p::mdns::{Mdns, MdnsConfig, MdnsEvent};
use libp2p::multiaddr::Protocol;
use libp2p::{
core::{
connection::{ConnectionId, ListenerId},
ConnectedPoint, Multiaddr, PeerId, PublicKey,
},
kad::{
handler::KademliaHandlerProto,
record::{
self,
store::{MemoryStore, RecordStore},
},
GetClosestPeersError, Kademlia, KademliaBucketInserts, KademliaConfig, KademliaEvent,
QueryId, QueryResult, Quorum, Record,
},
multiaddr::Protocol,
swarm::{
protocols_handler::multi::IntoMultiHandler, IntoProtocolsHandler, NetworkBehaviour,
NetworkBehaviourAction, PollParameters, ProtocolsHandler,
},
};
use log::{debug, info, trace, warn};
use std::{cmp, collections::{HashMap, HashSet, VecDeque}, io, num::NonZeroUsize, time::Duration};
use std::task::{Context, Poll};
use sp_core::hexdisplay::HexDisplay;
use std::{
cmp,
collections::{HashMap, HashSet, VecDeque},
io,
num::NonZeroUsize,
task::{Context, Poll},
time::Duration,
};
/// Maximum number of known external addresses that we will cache.
/// This only affects whether we will log whenever we (re-)discover
@@ -101,7 +116,7 @@ impl DiscoveryConfig {
discovery_only_if_under_num: std::u64::MAX,
enable_mdns: false,
kademlia_disjoint_query_paths: false,
protocol_ids: HashSet::new()
protocol_ids: HashSet::new(),
}
}
@@ -114,7 +129,7 @@ impl DiscoveryConfig {
/// Set custom nodes which never expire, e.g. bootstrap or reserved nodes.
pub fn with_user_defined<I>(&mut self, user_defined: I) -> &mut Self
where
I: IntoIterator<Item = (PeerId, Multiaddr)>
I: IntoIterator<Item = (PeerId, Multiaddr)>,
{
self.user_defined.extend(user_defined);
self
@@ -152,7 +167,7 @@ impl DiscoveryConfig {
pub fn add_protocol(&mut self, id: ProtocolId) -> &mut Self {
if self.protocol_ids.contains(&id) {
warn!(target: "sub-libp2p", "Discovery already registered for protocol {:?}", id);
return self;
return self
}
self.protocol_ids.insert(id);
@@ -181,7 +196,8 @@ impl DiscoveryConfig {
protocol_ids,
} = self;
let kademlias = protocol_ids.into_iter()
let kademlias = protocol_ids
.into_iter()
.map(|protocol_id| {
let proto_name = protocol_name_from_protocol_id(&protocol_id);
@@ -227,7 +243,7 @@ impl DiscoveryConfig {
allow_non_globals_in_dht,
known_external_addresses: LruHashSet::new(
NonZeroUsize::new(MAX_KNOWN_EXTERNAL_ADDRESSES)
.expect("value is a constant; constant is non-zero; qed.")
.expect("value is a constant; constant is non-zero; qed."),
),
}
}
@@ -305,7 +321,7 @@ impl DiscoveryBehaviour {
&mut self,
peer_id: &PeerId,
supported_protocols: impl Iterator<Item = impl AsRef<[u8]>>,
addr: Multiaddr
addr: Multiaddr,
) {
if !self.allow_non_globals_in_dht && !self.can_add_to_dht(&addr) {
log::trace!(target: "sub-libp2p", "Ignoring self-reported non-global address {} from {}.", addr, peer_id);
@@ -353,7 +369,8 @@ impl DiscoveryBehaviour {
for k in self.kademlias.values_mut() {
if let Err(e) = k.put_record(Record::new(key.clone(), value.clone()), Quorum::All) {
warn!(target: "sub-libp2p", "Libp2p => Failed to put record: {:?}", e);
self.pending_events.push_back(DiscoveryOut::ValuePutFailed(key.clone(), Duration::from_secs(0)));
self.pending_events
.push_back(DiscoveryOut::ValuePutFailed(key.clone(), Duration::from_secs(0)));
}
}
}
@@ -362,14 +379,16 @@ impl DiscoveryBehaviour {
///
/// Identifies Kademlia instances by their [`ProtocolId`] and kbuckets by the base 2 logarithm
/// of their lower bound.
pub fn num_entries_per_kbucket(&mut self) -> impl ExactSizeIterator<Item = (&ProtocolId, Vec<(u32, usize)>)> {
self.kademlias.iter_mut()
.map(|(id, kad)| {
let buckets = kad.kbuckets()
.map(|bucket| (bucket.range().0.ilog2().unwrap_or(0), bucket.iter().count()))
.collect();
(id, buckets)
})
pub fn num_entries_per_kbucket(
&mut self,
) -> impl ExactSizeIterator<Item = (&ProtocolId, Vec<(u32, usize)>)> {
self.kademlias.iter_mut().map(|(id, kad)| {
let buckets = kad
.kbuckets()
.map(|bucket| (bucket.range().0.ilog2().unwrap_or(0), bucket.iter().count()))
.collect();
(id, buckets)
})
}
/// Returns the number of records in the Kademlia record stores.
@@ -382,7 +401,9 @@ impl DiscoveryBehaviour {
}
/// Returns the total size in bytes of all the records in the Kademlia record stores.
pub fn kademlia_records_total_size(&mut self) -> impl ExactSizeIterator<Item = (&ProtocolId, usize)> {
pub fn kademlia_records_total_size(
&mut self,
) -> impl ExactSizeIterator<Item = (&ProtocolId, usize)> {
// Note that this code is ok only because we use a `MemoryStore`. If the records were
// for example stored on disk, this would load every single one of them every single time.
self.kademlias.iter_mut().map(|(id, kad)| {
@@ -394,7 +415,6 @@ impl DiscoveryBehaviour {
/// Can the given `Multiaddr` be put into the DHT?
///
/// This test is successful only for global IP addresses and DNS names.
//
// NB: Currently all DNS names are allowed and no check for TLD suffixes is done
// because the set of valid domains is highly dynamic and would require frequent
// updates, for example by utilising publicsuffix.org or IANA.
@@ -402,9 +422,9 @@ impl DiscoveryBehaviour {
let ip = match addr.iter().next() {
Some(Protocol::Ip4(ip)) => IpNetwork::from(ip),
Some(Protocol::Ip6(ip)) => IpNetwork::from(ip),
Some(Protocol::Dns(_)) | Some(Protocol::Dns4(_)) | Some(Protocol::Dns6(_))
=> return true,
_ => return false
Some(Protocol::Dns(_)) | Some(Protocol::Dns4(_)) | Some(Protocol::Dns6(_)) =>
return true,
_ => return false,
};
ip.is_global()
}
@@ -459,19 +479,24 @@ impl NetworkBehaviour for DiscoveryBehaviour {
type OutEvent = DiscoveryOut;
fn new_handler(&mut self) -> Self::ProtocolsHandler {
let iter = self.kademlias.iter_mut()
let iter = self
.kademlias
.iter_mut()
.map(|(p, k)| (p.clone(), NetworkBehaviour::new_handler(k)));
IntoMultiHandler::try_from_iter(iter)
.expect("There can be at most one handler per `ProtocolId` and \
IntoMultiHandler::try_from_iter(iter).expect(
"There can be at most one handler per `ProtocolId` and \
protocol names contain the `ProtocolId` so no two protocol \
names in `self.kademlias` can be equal which is the only error \
`try_from_iter` can return, therefore this call is guaranteed \
to succeed; qed")
to succeed; qed",
)
}
fn addresses_of_peer(&mut self, peer_id: &PeerId) -> Vec<Multiaddr> {
let mut list = self.user_defined.iter()
let mut list = self
.user_defined
.iter()
.filter_map(|(p, a)| if p == peer_id { Some(a.clone()) } else { None })
.collect::<Vec<_>>();
@@ -488,7 +513,7 @@ impl NetworkBehaviour for DiscoveryBehaviour {
list_to_filter.retain(|addr| {
if let Some(Protocol::Ip4(addr)) = addr.iter().next() {
if addr.is_private() {
return false;
return false
}
}
@@ -504,7 +529,12 @@ impl NetworkBehaviour for DiscoveryBehaviour {
list
}
fn inject_connection_established(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) {
fn inject_connection_established(
&mut self,
peer_id: &PeerId,
conn: &ConnectionId,
endpoint: &ConnectedPoint,
) {
self.num_connections += 1;
for k in self.kademlias.values_mut() {
NetworkBehaviour::inject_connection_established(k, peer_id, conn, endpoint)
@@ -517,7 +547,12 @@ impl NetworkBehaviour for DiscoveryBehaviour {
}
}
fn inject_connection_closed(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) {
fn inject_connection_closed(
&mut self,
peer_id: &PeerId,
conn: &ConnectionId,
endpoint: &ConnectedPoint,
) {
self.num_connections -= 1;
for k in self.kademlias.values_mut() {
NetworkBehaviour::inject_connection_closed(k, peer_id, conn, endpoint)
@@ -534,7 +569,7 @@ impl NetworkBehaviour for DiscoveryBehaviour {
&mut self,
peer_id: Option<&PeerId>,
addr: &Multiaddr,
error: &dyn std::error::Error
error: &dyn std::error::Error,
) {
for k in self.kademlias.values_mut() {
NetworkBehaviour::inject_addr_reach_failure(k, peer_id, addr, error)
@@ -556,8 +591,7 @@ impl NetworkBehaviour for DiscoveryBehaviour {
}
fn inject_new_external_addr(&mut self, addr: &Multiaddr) {
let new_addr = addr.clone()
.with(Protocol::P2p(self.local_peer_id.clone().into()));
let new_addr = addr.clone().with(Protocol::P2p(self.local_peer_id.clone().into()));
// NOTE: we might re-discover the same address multiple times
// in which case we just want to refrain from logging.
@@ -627,10 +661,10 @@ impl NetworkBehaviour for DiscoveryBehaviour {
<<Self::ProtocolsHandler as IntoProtocolsHandler>::Handler as ProtocolsHandler>::InEvent,
Self::OutEvent,
>,
> {
>{
// Immediately process the content of `discovered`.
if let Some(ev) = self.pending_events.pop_front() {
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev));
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev))
}
// Poll the stream that fires when we need to start a random Kademlia query.
@@ -657,12 +691,14 @@ impl NetworkBehaviour for DiscoveryBehaviour {
// Schedule the next random query with exponentially increasing delay,
// capped at 60 seconds.
*next_kad_random_query = Delay::new(self.duration_to_next_kad);
self.duration_to_next_kad = cmp::min(self.duration_to_next_kad * 2,
Duration::from_secs(60));
self.duration_to_next_kad =
cmp::min(self.duration_to_next_kad * 2, Duration::from_secs(60));
if actually_started {
let ev = DiscoveryOut::RandomKademliaStarted(self.kademlias.keys().cloned().collect());
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev));
let ev = DiscoveryOut::RandomKademliaStarted(
self.kademlias.keys().cloned().collect(),
);
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev))
}
}
}
@@ -674,86 +710,112 @@ impl NetworkBehaviour for DiscoveryBehaviour {
NetworkBehaviourAction::GenerateEvent(ev) => match ev {
KademliaEvent::RoutingUpdated { peer, .. } => {
let ev = DiscoveryOut::Discovered(peer);
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev));
}
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev))
},
KademliaEvent::UnroutablePeer { peer, .. } => {
let ev = DiscoveryOut::UnroutablePeer(peer);
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev));
}
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev))
},
KademliaEvent::RoutablePeer { peer, .. } => {
let ev = DiscoveryOut::Discovered(peer);
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev));
}
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev))
},
KademliaEvent::PendingRoutablePeer { .. } => {
// We are not interested in this event at the moment.
}
KademliaEvent::QueryResult { result: QueryResult::GetClosestPeers(res), .. } => {
match res {
Err(GetClosestPeersError::Timeout { key, peers }) => {
debug!(target: "sub-libp2p",
},
KademliaEvent::QueryResult {
result: QueryResult::GetClosestPeers(res),
..
} => match res {
Err(GetClosestPeersError::Timeout { key, peers }) => {
debug!(target: "sub-libp2p",
"Libp2p => Query for {:?} timed out with {} results",
HexDisplay::from(&key), peers.len());
},
Ok(ok) => {
trace!(target: "sub-libp2p",
},
Ok(ok) => {
trace!(target: "sub-libp2p",
"Libp2p => Query for {:?} yielded {:?} results",
HexDisplay::from(&ok.key), ok.peers.len());
if ok.peers.is_empty() && self.num_connections != 0 {
debug!(target: "sub-libp2p", "Libp2p => Random Kademlia query has yielded empty \
if ok.peers.is_empty() && self.num_connections != 0 {
debug!(target: "sub-libp2p", "Libp2p => Random Kademlia query has yielded empty \
results");
}
}
}
}
KademliaEvent::QueryResult { result: QueryResult::GetRecord(res), stats, .. } => {
},
},
KademliaEvent::QueryResult {
result: QueryResult::GetRecord(res),
stats,
..
} => {
let ev = match res {
Ok(ok) => {
let results = ok.records
let results = ok
.records
.into_iter()
.map(|r| (r.record.key, r.record.value))
.collect();
DiscoveryOut::ValueFound(results, stats.duration().unwrap_or_else(Default::default))
}
DiscoveryOut::ValueFound(
results,
stats.duration().unwrap_or_else(Default::default),
)
},
Err(e @ libp2p::kad::GetRecordError::NotFound { .. }) => {
trace!(target: "sub-libp2p",
"Libp2p => Failed to get record: {:?}", e);
DiscoveryOut::ValueNotFound(e.into_key(), stats.duration().unwrap_or_else(Default::default))
}
DiscoveryOut::ValueNotFound(
e.into_key(),
stats.duration().unwrap_or_else(Default::default),
)
},
Err(e) => {
debug!(target: "sub-libp2p",
"Libp2p => Failed to get record: {:?}", e);
DiscoveryOut::ValueNotFound(e.into_key(), stats.duration().unwrap_or_else(Default::default))
}
DiscoveryOut::ValueNotFound(
e.into_key(),
stats.duration().unwrap_or_else(Default::default),
)
},
};
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev));
}
KademliaEvent::QueryResult { result: QueryResult::PutRecord(res), stats, .. } => {
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev))
},
KademliaEvent::QueryResult {
result: QueryResult::PutRecord(res),
stats,
..
} => {
let ev = match res {
Ok(ok) => DiscoveryOut::ValuePut(ok.key, stats.duration().unwrap_or_else(Default::default)),
Ok(ok) => DiscoveryOut::ValuePut(
ok.key,
stats.duration().unwrap_or_else(Default::default),
),
Err(e) => {
debug!(target: "sub-libp2p",
"Libp2p => Failed to put record: {:?}", e);
DiscoveryOut::ValuePutFailed(e.into_key(), stats.duration().unwrap_or_else(Default::default))
}
DiscoveryOut::ValuePutFailed(
e.into_key(),
stats.duration().unwrap_or_else(Default::default),
)
},
};
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev));
}
KademliaEvent::QueryResult { result: QueryResult::RepublishRecord(res), .. } => {
match res {
Ok(ok) => debug!(target: "sub-libp2p",
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev))
},
KademliaEvent::QueryResult {
result: QueryResult::RepublishRecord(res),
..
} => match res {
Ok(ok) => debug!(target: "sub-libp2p",
"Libp2p => Record republished: {:?}",
ok.key),
Err(e) => debug!(target: "sub-libp2p",
Err(e) => debug!(target: "sub-libp2p",
"Libp2p => Republishing of record {:?} failed with: {:?}",
e.key(), e)
}
}
e.key(), e),
},
// We never start any other type of query.
e => {
warn!(target: "sub-libp2p", "Libp2p => Unhandled Kademlia event: {:?}", e)
}
}
},
},
NetworkBehaviourAction::DialAddress { address } =>
return Poll::Ready(NetworkBehaviourAction::DialAddress { address }),
NetworkBehaviourAction::DialPeer { peer_id, condition } =>
@@ -762,10 +824,13 @@ impl NetworkBehaviour for DiscoveryBehaviour {
return Poll::Ready(NetworkBehaviourAction::NotifyHandler {
peer_id,
handler,
event: (pid.clone(), event)
event: (pid.clone(), event),
}),
NetworkBehaviourAction::ReportObservedAddr { address, score } =>
return Poll::Ready(NetworkBehaviourAction::ReportObservedAddr { address, score }),
return Poll::Ready(NetworkBehaviourAction::ReportObservedAddr {
address,
score,
}),
}
}
}
@@ -774,29 +839,30 @@ impl NetworkBehaviour for DiscoveryBehaviour {
#[cfg(not(target_os = "unknown"))]
while let Poll::Ready(ev) = self.mdns.poll(cx, params) {
match ev {
NetworkBehaviourAction::GenerateEvent(event) => {
match event {
MdnsEvent::Discovered(list) => {
if self.num_connections >= self.discovery_only_if_under_num {
continue;
}
NetworkBehaviourAction::GenerateEvent(event) => match event {
MdnsEvent::Discovered(list) => {
if self.num_connections >= self.discovery_only_if_under_num {
continue
}
self.pending_events.extend(list.map(|(peer_id, _)| DiscoveryOut::Discovered(peer_id)));
if let Some(ev) = self.pending_events.pop_front() {
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev));
}
},
MdnsEvent::Expired(_) => {}
}
self.pending_events
.extend(list.map(|(peer_id, _)| DiscoveryOut::Discovered(peer_id)));
if let Some(ev) = self.pending_events.pop_front() {
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev))
}
},
MdnsEvent::Expired(_) => {},
},
NetworkBehaviourAction::DialAddress { address } =>
return Poll::Ready(NetworkBehaviourAction::DialAddress { address }),
NetworkBehaviourAction::DialPeer { peer_id, condition } =>
return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }),
NetworkBehaviourAction::NotifyHandler { event, .. } =>
match event {}, // `event` is an enum with no variant
NetworkBehaviourAction::NotifyHandler { event, .. } => match event {}, /* `event` is an enum with no variant */
NetworkBehaviourAction::ReportObservedAddr { address, score } =>
return Poll::Ready(NetworkBehaviourAction::ReportObservedAddr { address, score }),
return Poll::Ready(NetworkBehaviourAction::ReportObservedAddr {
address,
score,
}),
}
}
@@ -839,15 +905,14 @@ impl MdnsWrapper {
) -> Poll<NetworkBehaviourAction<void::Void, MdnsEvent>> {
loop {
match self {
MdnsWrapper::Instantiating(fut) => {
MdnsWrapper::Instantiating(fut) =>
*self = match futures::ready!(fut.as_mut().poll(cx)) {
Ok(mdns) => MdnsWrapper::Ready(mdns),
Err(err) => {
warn!(target: "sub-libp2p", "Failed to initialize mDNS: {:?}", err);
MdnsWrapper::Disabled
},
}
}
},
MdnsWrapper::Ready(mdns) => return mdns.poll(cx, params),
MdnsWrapper::Disabled => return Poll::Pending,
}
@@ -857,17 +922,20 @@ impl MdnsWrapper {
#[cfg(test)]
mod tests {
use super::{protocol_name_from_protocol_id, DiscoveryConfig, DiscoveryOut};
use crate::config::ProtocolId;
use futures::prelude::*;
use libp2p::identity::Keypair;
use libp2p::{Multiaddr, PeerId};
use libp2p::core::upgrade;
use libp2p::core::transport::{Transport, MemoryTransport};
use libp2p::noise;
use libp2p::swarm::Swarm;
use libp2p::yamux;
use libp2p::{
core::{
transport::{MemoryTransport, Transport},
upgrade,
},
identity::Keypair,
noise,
swarm::Swarm,
yamux, Multiaddr, PeerId,
};
use std::{collections::HashSet, task::Poll};
use super::{DiscoveryConfig, DiscoveryOut, protocol_name_from_protocol_id};
#[test]
fn discovery_working() {
@@ -876,50 +944,56 @@ mod tests {
// Build swarms whose behaviour is `DiscoveryBehaviour`, each aware of
// the first swarm via `with_user_defined`.
let mut swarms = (0..25).map(|i| {
let keypair = Keypair::generate_ed25519();
let mut swarms = (0..25)
.map(|i| {
let keypair = Keypair::generate_ed25519();
let noise_keys = noise::Keypair::<noise::X25519Spec>::new()
.into_authentic(&keypair)
.unwrap();
let noise_keys =
noise::Keypair::<noise::X25519Spec>::new().into_authentic(&keypair).unwrap();
let transport = MemoryTransport
.upgrade(upgrade::Version::V1)
.authenticate(noise::NoiseConfig::xx(noise_keys).into_authenticated())
.multiplex(yamux::YamuxConfig::default())
.boxed();
let transport = MemoryTransport
.upgrade(upgrade::Version::V1)
.authenticate(noise::NoiseConfig::xx(noise_keys).into_authenticated())
.multiplex(yamux::YamuxConfig::default())
.boxed();
let behaviour = {
let mut config = DiscoveryConfig::new(keypair.public());
config.with_user_defined(first_swarm_peer_id_and_addr.clone())
.allow_private_ipv4(true)
.allow_non_globals_in_dht(true)
.discovery_limit(50)
.add_protocol(protocol_id.clone());
let behaviour = {
let mut config = DiscoveryConfig::new(keypair.public());
config
.with_user_defined(first_swarm_peer_id_and_addr.clone())
.allow_private_ipv4(true)
.allow_non_globals_in_dht(true)
.discovery_limit(50)
.add_protocol(protocol_id.clone());
config.finish()
};
config.finish()
};
let mut swarm = Swarm::new(transport, behaviour, keypair.public().into_peer_id());
let listen_addr: Multiaddr = format!("/memory/{}", rand::random::<u64>()).parse().unwrap();
let mut swarm = Swarm::new(transport, behaviour, keypair.public().into_peer_id());
let listen_addr: Multiaddr =
format!("/memory/{}", rand::random::<u64>()).parse().unwrap();
if i == 0 {
first_swarm_peer_id_and_addr = Some((keypair.public().into_peer_id(), listen_addr.clone()))
}
if i == 0 {
first_swarm_peer_id_and_addr =
Some((keypair.public().into_peer_id(), listen_addr.clone()))
}
swarm.listen_on(listen_addr.clone()).unwrap();
(swarm, listen_addr)
}).collect::<Vec<_>>();
swarm.listen_on(listen_addr.clone()).unwrap();
(swarm, listen_addr)
})
.collect::<Vec<_>>();
// Build a `Vec<HashSet<PeerId>>` with the list of nodes remaining to be discovered.
let mut to_discover = (0..swarms.len()).map(|n| {
(0..swarms.len())
// Skip the first swarm as all other swarms already know it.
.skip(1)
.filter(|p| *p != n)
.map(|p| Swarm::local_peer_id(&swarms[p].0).clone())
.collect::<HashSet<_>>()
}).collect::<Vec<_>>();
let mut to_discover = (0..swarms.len())
.map(|n| {
(0..swarms.len())
// Skip the first swarm as all other swarms already know it.
.skip(1)
.filter(|p| *p != n)
.map(|p| Swarm::local_peer_id(&swarms[p].0).clone())
.collect::<HashSet<_>>()
})
.collect::<Vec<_>>();
let fut = futures::future::poll_fn(move |cx| {
'polling: loop {
@@ -927,13 +1001,17 @@ mod tests {
match swarms[swarm_n].0.poll_next_unpin(cx) {
Poll::Ready(Some(e)) => {
match e {
DiscoveryOut::UnroutablePeer(other) | DiscoveryOut::Discovered(other) => {
DiscoveryOut::UnroutablePeer(other) |
DiscoveryOut::Discovered(other) => {
// Call `add_self_reported_address` to simulate identify happening.
let addr = swarms.iter().find_map(|(s, a)|
if s.behaviour().local_peer_id == other {
Some(a.clone())
} else {
None
let addr = swarms
.iter()
.find_map(|(s, a)| {
if s.behaviour().local_peer_id == other {
Some(a.clone())
} else {
None
}
})
.unwrap();
swarms[swarm_n].0.behaviour_mut().add_self_reported_address(
@@ -945,11 +1023,13 @@ mod tests {
to_discover[swarm_n].remove(&other);
},
DiscoveryOut::RandomKademliaStarted(_) => {},
e => {panic!("Unexpected event: {:?}", e)},
e => {
panic!("Unexpected event: {:?}", e)
},
}
continue 'polling
}
_ => {}
},
_ => {},
}
}
break
@@ -973,7 +1053,8 @@ mod tests {
let mut discovery = {
let keypair = Keypair::generate_ed25519();
let mut config = DiscoveryConfig::new(keypair.public());
config.allow_private_ipv4(true)
config
.allow_private_ipv4(true)
.allow_non_globals_in_dht(true)
.discovery_limit(50)
.add_protocol(supported_protocol_id.clone());
@@ -992,7 +1073,8 @@ mod tests {
for kademlia in discovery.kademlias.values_mut() {
assert!(
kademlia.kbucket(remote_peer_id.clone())
kademlia
.kbucket(remote_peer_id.clone())
.expect("Remote peer id not to be equal to local peer id.")
.is_empty(),
"Expect peer with unsupported protocol not to be added."
@@ -1009,7 +1091,8 @@ mod tests {
for kademlia in discovery.kademlias.values_mut() {
assert_eq!(
1,
kademlia.kbucket(remote_peer_id.clone())
kademlia
.kbucket(remote_peer_id.clone())
.expect("Remote peer id not to be equal to local peer id.")
.num_entries(),
"Expect peer with supported protocol to be added."
@@ -1025,7 +1108,8 @@ mod tests {
let mut discovery = {
let keypair = Keypair::generate_ed25519();
let mut config = DiscoveryConfig::new(keypair.public());
config.allow_private_ipv4(true)
config
.allow_private_ipv4(true)
.allow_non_globals_in_dht(true)
.discovery_limit(50)
.add_protocol(protocol_a.clone())
@@ -1045,17 +1129,20 @@ mod tests {
assert_eq!(
1,
discovery.kademlias.get_mut(&protocol_a)
discovery
.kademlias
.get_mut(&protocol_a)
.expect("Kademlia instance to exist.")
.kbucket(remote_peer_id.clone())
.expect("Remote peer id not to be equal to local peer id.")
.num_entries(),
"Expected remote peer to be added to `protocol_a` Kademlia instance.",
);
assert!(
discovery.kademlias.get_mut(&protocol_b)
discovery
.kademlias
.get_mut(&protocol_b)
.expect("Kademlia instance to exist.")
.kbucket(remote_peer_id.clone())
.expect("Remote peer id not to be equal to local peer id.")
+3 -3
View File
@@ -19,7 +19,7 @@
//! Substrate network possible errors.
use crate::config::TransportConfig;
use libp2p::{PeerId, Multiaddr};
use libp2p::{Multiaddr, PeerId};
use std::{borrow::Cow, fmt};
@@ -38,7 +38,7 @@ pub enum Error {
fmt = "The same bootnode (`{}`) is registered with two different peer ids: `{}` and `{}`",
address,
first_id,
second_id,
second_id
)]
DuplicateBootnode {
/// The address of the bootnode.
@@ -53,7 +53,7 @@ pub enum Error {
/// The network addresses are invalid because they don't match the transport.
#[display(
fmt = "The following addresses are invalid because they don't match the transport: {:?}",
addresses,
addresses
)]
AddressesForAnotherTransport {
/// Transport used.
+11 -9
View File
@@ -243,13 +243,12 @@
//! - Calling `trigger_repropagate` when a transaction is added to the pool.
//!
//! More precise usage details are still being worked on and will likely change in the future.
//!
mod behaviour;
mod chain;
mod peer_info;
mod discovery;
mod on_demand_layer;
mod peer_info;
mod protocol;
mod request_responses;
mod schema;
@@ -257,22 +256,25 @@ mod service;
mod transport;
mod utils;
pub mod block_request_handler;
pub mod bitswap;
pub mod light_client_requests;
pub mod state_request_handler;
pub mod block_request_handler;
pub mod config;
pub mod error;
pub mod light_client_requests;
pub mod network_state;
pub mod state_request_handler;
pub mod transactions;
#[doc(inline)]
pub use libp2p::{multiaddr, Multiaddr, PeerId};
pub use protocol::{event::{DhtEvent, Event, ObservedRole}, PeerInfo};
pub use protocol::sync::{SyncState, StateDownloadProgress};
pub use protocol::{
event::{DhtEvent, Event, ObservedRole},
sync::{StateDownloadProgress, SyncState},
PeerInfo,
};
pub use service::{
NetworkService, NetworkWorker, RequestFailure, OutboundFailure, NotificationSender,
NotificationSenderReady, IfDisconnected,
IfDisconnected, NetworkService, NetworkWorker, NotificationSender, NotificationSenderReady,
OutboundFailure, RequestFailure,
};
pub use sc_peerset::ReputationChange;
@@ -18,13 +18,12 @@
//! Helpers for outgoing and incoming light client requests.
/// For outgoing light client requests.
pub mod sender;
/// For incoming light client requests.
pub mod handler;
/// For outgoing light client requests.
pub mod sender;
use crate::config::ProtocolId;
use crate::request_responses::ProtocolConfig;
use crate::{config::ProtocolId, request_responses::ProtocolConfig};
use std::time::Duration;
@@ -51,24 +50,30 @@ pub fn generate_protocol_config(protocol_id: &ProtocolId) -> ProtocolConfig {
#[cfg(test)]
mod tests {
use super::*;
use crate::request_responses::IncomingRequest;
use crate::config::ProtocolId;
use crate::{config::ProtocolId, request_responses::IncomingRequest};
use assert_matches::assert_matches;
use futures::executor::{block_on, LocalPool};
use futures::task::Spawn;
use futures::{channel::oneshot, prelude::*};
use futures::{
channel::oneshot,
executor::{block_on, LocalPool},
prelude::*,
task::Spawn,
};
use libp2p::PeerId;
use sc_client_api::StorageProof;
use sc_client_api::light::{RemoteCallRequest, RemoteChangesRequest, RemoteHeaderRequest};
use sc_client_api::light::{self, RemoteReadRequest, RemoteBodyRequest, ChangesProof};
use sc_client_api::{FetchChecker, RemoteReadChildRequest};
use sc_client_api::{
light::{
self, ChangesProof, RemoteBodyRequest, RemoteCallRequest, RemoteChangesRequest,
RemoteHeaderRequest, RemoteReadRequest,
},
FetchChecker, RemoteReadChildRequest, StorageProof,
};
use sp_blockchain::Error as ClientError;
use sp_core::storage::ChildInfo;
use sp_runtime::generic::Header;
use sp_runtime::traits::{BlakeTwo256, Block as BlockT, NumberFor};
use std::collections::HashMap;
use std::sync::Arc;
use sp_runtime::{
generic::Header,
traits::{BlakeTwo256, Block as BlockT, NumberFor},
};
use std::{collections::HashMap, sync::Arc};
pub struct DummyFetchChecker<B> {
pub ok: bool,
@@ -94,12 +99,7 @@ mod tests {
_: StorageProof,
) -> Result<HashMap<Vec<u8>, Option<Vec<u8>>>, ClientError> {
match self.ok {
true => Ok(request
.keys
.iter()
.cloned()
.map(|k| (k, Some(vec![42])))
.collect()),
true => Ok(request.keys.iter().cloned().map(|k| (k, Some(vec![42]))).collect()),
false => Err(ClientError::Backend("Test error".into())),
}
}
@@ -110,12 +110,7 @@ mod tests {
_: StorageProof,
) -> Result<HashMap<Vec<u8>, Option<Vec<u8>>>, ClientError> {
match self.ok {
true => Ok(request
.keys
.iter()
.cloned()
.map(|k| (k, Some(vec![42])))
.collect()),
true => Ok(request.keys.iter().cloned().map(|k| (k, Some(vec![42]))).collect()),
false => Err(ClientError::Backend("Test error".into())),
}
}
@@ -184,7 +179,8 @@ mod tests {
fn send_receive(request: sender::Request<Block>, pool: &LocalPool) {
let client = Arc::new(substrate_test_runtime_client::new());
let (handler, protocol_config) = handler::LightClientRequestHandler::new(&protocol_id(), client);
let (handler, protocol_config) =
handler::LightClientRequestHandler::new(&protocol_id(), client);
pool.spawner().spawn_obj(handler.run().boxed().into()).unwrap();
let (_peer_set, peer_set_handle) = peerset();
@@ -199,18 +195,28 @@ mod tests {
sender.inject_connected(PeerId::random());
sender.request(request).unwrap();
let sender::OutEvent::SendRequest { pending_response, request, .. } = block_on(sender.next()).unwrap();
let sender::OutEvent::SendRequest { pending_response, request, .. } =
block_on(sender.next()).unwrap();
let (tx, rx) = oneshot::channel();
block_on(protocol_config.inbound_queue.unwrap().send(IncomingRequest {
peer: PeerId::random(),
payload: request,
pending_response: tx,
})).unwrap();
pool.spawner().spawn_obj(async move {
pending_response.send(Ok(rx.await.unwrap().result.unwrap())).unwrap();
}.boxed().into()).unwrap();
}))
.unwrap();
pool.spawner()
.spawn_obj(
async move {
pending_response.send(Ok(rx.await.unwrap().result.unwrap())).unwrap();
}
.boxed()
.into(),
)
.unwrap();
pool.spawner().spawn_obj(sender.for_each(|_| future::ready(())).boxed().into()).unwrap();
pool.spawner()
.spawn_obj(sender.for_each(|_| future::ready(())).boxed().into())
.unwrap();
}
#[test]
@@ -225,10 +231,7 @@ mod tests {
};
let mut pool = LocalPool::new();
send_receive(sender::Request::Call {
request,
sender: chan.0,
}, &pool);
send_receive(sender::Request::Call { request, sender: chan.0 }, &pool);
assert_eq!(vec![42], pool.run_until(chan.1).unwrap().unwrap());
// ^--- from `DummyFetchChecker::check_execution_proof`
}
@@ -243,17 +246,10 @@ mod tests {
retry_count: None,
};
let mut pool = LocalPool::new();
send_receive(sender::Request::Read {
request,
sender: chan.0,
}, &pool);
send_receive(sender::Request::Read { request, sender: chan.0 }, &pool);
assert_eq!(
Some(vec![42]),
pool.run_until(chan.1)
.unwrap()
.unwrap()
.remove(&b":key"[..])
.unwrap()
pool.run_until(chan.1).unwrap().unwrap().remove(&b":key"[..]).unwrap()
);
// ^--- from `DummyFetchChecker::check_read_proof`
}
@@ -270,17 +266,10 @@ mod tests {
retry_count: None,
};
let mut pool = LocalPool::new();
send_receive(sender::Request::ReadChild {
request,
sender: chan.0,
}, &pool);
send_receive(sender::Request::ReadChild { request, sender: chan.0 }, &pool);
assert_eq!(
Some(vec![42]),
pool.run_until(chan.1)
.unwrap()
.unwrap()
.remove(&b":key"[..])
.unwrap()
pool.run_until(chan.1).unwrap().unwrap().remove(&b":key"[..]).unwrap()
);
// ^--- from `DummyFetchChecker::check_read_child_proof`
}
@@ -295,15 +284,9 @@ mod tests {
retry_count: None,
};
let mut pool = LocalPool::new();
send_receive(sender::Request::Header {
request,
sender: chan.0,
}, &pool);
send_receive(sender::Request::Header { request, sender: chan.0 }, &pool);
// The remote does not know block 1:
assert_matches!(
pool.run_until(chan.1).unwrap(),
Err(ClientError::RemoteFetchFailed)
);
assert_matches!(pool.run_until(chan.1).unwrap(), Err(ClientError::RemoteFetchFailed));
}
#[test]
@@ -324,10 +307,7 @@ mod tests {
retry_count: None,
};
let mut pool = LocalPool::new();
send_receive(sender::Request::Changes {
request,
sender: chan.0,
}, &pool);
send_receive(sender::Request::Changes { request, sender: chan.0 }, &pool);
assert_eq!(vec![(100, 2)], pool.run_until(chan.1).unwrap().unwrap());
// ^--- from `DummyFetchChecker::check_changes_proof`
}
@@ -22,34 +22,27 @@
//! [`crate::request_responses::RequestResponsesBehaviour`] with
//! [`LightClientRequestHandler`](handler::LightClientRequestHandler).
use codec::{self, Encode, Decode};
use crate::{
chain::Client,
config::ProtocolId,
schema,
PeerId,
request_responses::{IncomingRequest, OutgoingResponse, ProtocolConfig},
schema, PeerId,
};
use crate::request_responses::{IncomingRequest, OutgoingResponse, ProtocolConfig};
use futures::{channel::mpsc, prelude::*};
use codec::{self, Decode, Encode};
use futures::{channel::mpsc, prelude::*};
use log::{debug, trace};
use prost::Message;
use sc_client_api::{
StorageProof,
light
};
use sc_client_api::{light, StorageProof};
use sc_peerset::ReputationChange;
use sp_core::{
storage::{ChildInfo, ChildType,StorageKey, PrefixedStorageKey},
hexdisplay::HexDisplay,
storage::{ChildInfo, ChildType, PrefixedStorageKey, StorageKey},
};
use sp_runtime::{
traits::{Block, Zero},
generic::BlockId,
traits::{Block, Zero},
};
use std::{
collections::{BTreeMap},
sync::Arc,
};
use log::{trace, debug};
use std::{collections::BTreeMap, sync::Arc};
const LOG_TARGET: &str = "light-client-request-handler";
@@ -62,10 +55,7 @@ pub struct LightClientRequestHandler<B: Block> {
impl<B: Block> LightClientRequestHandler<B> {
/// Create a new [`crate::block_request_handler::BlockRequestHandler`].
pub fn new(
protocol_id: &ProtocolId,
client: Arc<dyn Client<B>>,
) -> (Self, ProtocolConfig) {
pub fn new(protocol_id: &ProtocolId, client: Arc<dyn Client<B>>) -> (Self, ProtocolConfig) {
// For now due to lack of data on light client request handling in production systems, this
// value is chosen to match the block request limit.
let (tx, request_receiver) = mpsc::channel(20);
@@ -86,7 +76,7 @@ impl<B: Block> LightClientRequestHandler<B> {
let response = OutgoingResponse {
result: Ok(response_data),
reputation_changes: Vec::new(),
sent_feedback: None
sent_feedback: None,
};
match pending_response.send(response) {
@@ -98,35 +88,36 @@ impl<B: Block> LightClientRequestHandler<B> {
Err(_) => debug!(
target: LOG_TARGET,
"Failed to handle light client request from {}: {}",
peer, HandleRequestError::SendResponse,
peer,
HandleRequestError::SendResponse,
),
};
} ,
},
Err(e) => {
debug!(
target: LOG_TARGET,
"Failed to handle light client request from {}: {}",
peer, e,
"Failed to handle light client request from {}: {}", peer, e,
);
let reputation_changes = match e {
HandleRequestError::BadRequest(_) => {
vec![ReputationChange::new(-(1 << 12), "bad request")]
}
},
_ => Vec::new(),
};
let response = OutgoingResponse {
result: Err(()),
reputation_changes,
sent_feedback: None
sent_feedback: None,
};
if pending_response.send(response).is_err() {
debug!(
target: LOG_TARGET,
"Failed to handle light client request from {}: {}",
peer, HandleRequestError::SendResponse,
peer,
HandleRequestError::SendResponse,
);
};
},
@@ -134,7 +125,6 @@ impl<B: Block> LightClientRequestHandler<B> {
}
}
fn handle_request(
&mut self,
peer: PeerId,
@@ -153,9 +143,8 @@ impl<B: Block> LightClientRequestHandler<B> {
self.on_remote_read_child_request(&peer, r)?,
Some(schema::v1::light::request::Request::RemoteChangesRequest(r)) =>
self.on_remote_changes_request(&peer, r)?,
None => {
return Err(HandleRequestError::BadRequest("Remote request without request data."));
}
None =>
return Err(HandleRequestError::BadRequest("Remote request without request data.")),
};
let mut data = Vec::new();
@@ -171,24 +160,30 @@ impl<B: Block> LightClientRequestHandler<B> {
) -> Result<schema::v1::light::Response, HandleRequestError> {
log::trace!(
"Remote call request from {} ({} at {:?}).",
peer, request.method, request.block,
peer,
request.method,
request.block,
);
let block = Decode::decode(&mut request.block.as_ref())?;
let proof = match self.client.execution_proof(
&BlockId::Hash(block),
&request.method, &request.data,
) {
Ok((_, proof)) => proof,
Err(e) => {
log::trace!(
"remote call request from {} ({} at {:?}) failed with: {}",
peer, request.method, request.block, e,
);
StorageProof::empty()
}
};
let proof =
match self
.client
.execution_proof(&BlockId::Hash(block), &request.method, &request.data)
{
Ok((_, proof)) => proof,
Err(e) => {
log::trace!(
"remote call request from {} ({} at {:?}) failed with: {}",
peer,
request.method,
request.block,
e,
);
StorageProof::empty()
},
};
let response = {
let r = schema::v1::light::RemoteCallResponse { proof: proof.encode() };
@@ -210,23 +205,28 @@ impl<B: Block> LightClientRequestHandler<B> {
log::trace!(
"Remote read request from {} ({} at {:?}).",
peer, fmt_keys(request.keys.first(), request.keys.last()), request.block,
peer,
fmt_keys(request.keys.first(), request.keys.last()),
request.block,
);
let block = Decode::decode(&mut request.block.as_ref())?;
let proof = match self.client.read_proof(
&BlockId::Hash(block),
&mut request.keys.iter().map(AsRef::as_ref),
) {
let proof = match self
.client
.read_proof(&BlockId::Hash(block), &mut request.keys.iter().map(AsRef::as_ref))
{
Ok(proof) => proof,
Err(error) => {
log::trace!(
"remote read request from {} ({} at {:?}) failed with: {}",
peer, fmt_keys(request.keys.first(), request.keys.last()), request.block, error,
peer,
fmt_keys(request.keys.first(), request.keys.last()),
request.block,
error,
);
StorageProof::empty()
}
},
};
let response = {
@@ -262,11 +262,13 @@ impl<B: Block> LightClientRequestHandler<B> {
Some((ChildType::ParentKeyId, storage_key)) => Ok(ChildInfo::new_default(storage_key)),
None => Err(sp_blockchain::Error::InvalidChildStorageKey),
};
let proof = match child_info.and_then(|child_info| self.client.read_child_proof(
&BlockId::Hash(block),
&child_info,
&mut request.keys.iter().map(AsRef::as_ref)
)) {
let proof = match child_info.and_then(|child_info| {
self.client.read_child_proof(
&BlockId::Hash(block),
&child_info,
&mut request.keys.iter().map(AsRef::as_ref),
)
}) {
Ok(proof) => proof,
Err(error) => {
log::trace!(
@@ -278,7 +280,7 @@ impl<B: Block> LightClientRequestHandler<B> {
error,
);
StorageProof::empty()
}
},
};
let response = {
@@ -302,10 +304,12 @@ impl<B: Block> LightClientRequestHandler<B> {
Err(error) => {
log::trace!(
"Remote header proof request from {} ({:?}) failed with: {}.",
peer, request.block, error
peer,
request.block,
error
);
(Default::default(), StorageProof::empty())
}
},
};
let response = {
@@ -325,7 +329,11 @@ impl<B: Block> LightClientRequestHandler<B> {
"Remote changes proof request from {} for key {} ({:?}..{:?}).",
peer,
if !request.storage_key.is_empty() {
format!("{} : {}", HexDisplay::from(&request.storage_key), HexDisplay::from(&request.key))
format!(
"{} : {}",
HexDisplay::from(&request.storage_key),
HexDisplay::from(&request.key)
)
} else {
HexDisplay::from(&request.key).to_string()
},
@@ -344,10 +352,11 @@ impl<B: Block> LightClientRequestHandler<B> {
Some(PrefixedStorageKey::new_ref(&request.storage_key))
};
let proof = match self.client.key_changes_proof(first, last, min, max, storage_key, &key) {
Ok(proof) => proof,
Err(error) => {
log::trace!(
let proof =
match self.client.key_changes_proof(first, last, min, max, storage_key, &key) {
Ok(proof) => proof,
Err(error) => {
log::trace!(
"Remote changes proof request from {} for key {} ({:?}..{:?}) failed with: {}.",
peer,
format!("{} : {}", HexDisplay::from(&request.storage_key), HexDisplay::from(&key.0)),
@@ -356,20 +365,22 @@ impl<B: Block> LightClientRequestHandler<B> {
error,
);
light::ChangesProof::<B::Header> {
max_block: Zero::zero(),
proof: Vec::new(),
roots: BTreeMap::new(),
roots_proof: StorageProof::empty(),
}
}
};
light::ChangesProof::<B::Header> {
max_block: Zero::zero(),
proof: Vec::new(),
roots: BTreeMap::new(),
roots_proof: StorageProof::empty(),
}
},
};
let response = {
let r = schema::v1::light::RemoteChangesResponse {
max: proof.max_block.encode(),
proof: proof.proof,
roots: proof.roots.into_iter()
roots: proof
.roots
.into_iter()
.map(|(k, v)| schema::v1::light::Pair { fst: k.encode(), snd: v.encode() })
.collect(),
roots_proof: proof.roots_proof.encode(),
@@ -29,28 +29,21 @@
//! 3. Wait for the response and forward the response via the [`futures::channel::oneshot::Sender`] provided earlier
//! with [`LightClientRequestSender::request`](sender::LightClientRequestSender::request).
use codec::{self, Encode, Decode};
use crate::{
config::ProtocolId,
protocol::message::{BlockAttributes},
schema,
PeerId,
protocol::message::BlockAttributes,
request_responses::{OutboundFailure, RequestFailure},
schema, PeerId,
};
use crate::request_responses::{RequestFailure, OutboundFailure};
use futures::{channel::{oneshot}, future::BoxFuture, prelude::*, stream::FuturesUnordered};
use codec::{self, Decode, Encode};
use futures::{channel::oneshot, future::BoxFuture, prelude::*, stream::FuturesUnordered};
use prost::Message;
use sc_client_api::{
light::{
self, RemoteBodyRequest,
}
};
use sc_client_api::light::{self, RemoteBodyRequest};
use sc_peerset::ReputationChange;
use sp_blockchain::{Error as ClientError};
use sp_runtime::{
traits::{Block, Header, NumberFor},
};
use sp_blockchain::Error as ClientError;
use sp_runtime::traits::{Block, Header, NumberFor};
use std::{
collections::{BTreeMap, VecDeque, HashMap},
collections::{BTreeMap, HashMap, VecDeque},
pin::Pin,
sync::Arc,
task::{Context, Poll},
@@ -60,9 +53,11 @@ mod rep {
use super::*;
/// Reputation change for a peer when a request timed out.
pub const TIMEOUT: ReputationChange = ReputationChange::new(-(1 << 8), "light client request timeout");
pub const TIMEOUT: ReputationChange =
ReputationChange::new(-(1 << 8), "light client request timeout");
/// Reputation change for a peer when a request is refused.
pub const REFUSED: ReputationChange = ReputationChange::new(-(1 << 8), "light client request refused");
pub const REFUSED: ReputationChange =
ReputationChange::new(-(1 << 8), "light client request refused");
}
/// Configuration options for [`LightClientRequestSender`].
@@ -95,9 +90,12 @@ pub struct LightClientRequestSender<B: Block> {
/// Pending (local) requests.
pending_requests: VecDeque<PendingRequest<B>>,
/// Requests on their way to remote peers.
sent_requests: FuturesUnordered<BoxFuture<
'static, (SentRequest<B>, Result<Result<Vec<u8>, RequestFailure>, oneshot::Canceled>),
>>,
sent_requests: FuturesUnordered<
BoxFuture<
'static,
(SentRequest<B>, Result<Result<Vec<u8>, RequestFailure>, oneshot::Canceled>),
>,
>,
/// Handle to use for reporting misbehaviour of peers.
peerset: sc_peerset::PeersetHandle,
}
@@ -121,11 +119,7 @@ impl<B: Block> PendingRequest<B> {
}
fn into_sent(self, peer_id: PeerId) -> SentRequest<B> {
SentRequest {
attempts_left: self.attempts_left,
request: self.request,
peer: peer_id,
}
SentRequest { attempts_left: self.attempts_left, request: self.request, peer: peer_id }
}
}
@@ -142,10 +136,7 @@ struct SentRequest<B: Block> {
impl<B: Block> SentRequest<B> {
fn into_pending(self) -> PendingRequest<B> {
PendingRequest {
attempts_left: self.attempts_left,
request: self.request,
}
PendingRequest { attempts_left: self.attempts_left, request: self.request }
}
}
@@ -206,7 +197,7 @@ where
peer: PeerId,
request: &Request<B>,
response: Response,
) -> Result<Reply<B>, Error> {
) -> Result<Reply<B>, Error> {
log::trace!("response from {}", peer);
match response {
Response::Light(r) => self.on_response_light(request, r),
@@ -222,27 +213,26 @@ where
use schema::v1::light::response::Response;
match response.response {
Some(Response::RemoteCallResponse(response)) =>
if let Request::Call { request , .. } = request {
if let Request::Call { request, .. } = request {
let proof = Decode::decode(&mut response.proof.as_ref())?;
let reply = self.checker.check_execution_proof(request, proof)?;
Ok(Reply::VecU8(reply))
} else {
Err(Error::UnexpectedResponse)
}
Some(Response::RemoteReadResponse(response)) =>
match request {
Request::Read { request, .. } => {
let proof = Decode::decode(&mut response.proof.as_ref())?;
let reply = self.checker.check_read_proof(&request, proof)?;
Ok(Reply::MapVecU8OptVecU8(reply))
}
Request::ReadChild { request, .. } => {
let proof = Decode::decode(&mut response.proof.as_ref())?;
let reply = self.checker.check_read_child_proof(&request, proof)?;
Ok(Reply::MapVecU8OptVecU8(reply))
}
_ => Err(Error::UnexpectedResponse)
}
},
Some(Response::RemoteReadResponse(response)) => match request {
Request::Read { request, .. } => {
let proof = Decode::decode(&mut response.proof.as_ref())?;
let reply = self.checker.check_read_proof(&request, proof)?;
Ok(Reply::MapVecU8OptVecU8(reply))
},
Request::ReadChild { request, .. } => {
let proof = Decode::decode(&mut response.proof.as_ref())?;
let reply = self.checker.check_read_child_proof(&request, proof)?;
Ok(Reply::MapVecU8OptVecU8(reply))
},
_ => Err(Error::UnexpectedResponse),
},
Some(Response::RemoteChangesResponse(response)) =>
if let Request::Changes { request, .. } = request {
let max_block = Decode::decode(&mut response.max.as_ref())?;
@@ -256,31 +246,33 @@ where
}
r
};
let reply = self.checker.check_changes_proof(&request, light::ChangesProof {
max_block,
proof: response.proof,
roots,
roots_proof,
})?;
let reply = self.checker.check_changes_proof(
&request,
light::ChangesProof {
max_block,
proof: response.proof,
roots,
roots_proof,
},
)?;
Ok(Reply::VecNumberU32(reply))
} else {
Err(Error::UnexpectedResponse)
}
},
Some(Response::RemoteHeaderResponse(response)) =>
if let Request::Header { request, .. } = request {
let header =
if response.header.is_empty() {
None
} else {
Some(Decode::decode(&mut response.header.as_ref())?)
};
let header = if response.header.is_empty() {
None
} else {
Some(Decode::decode(&mut response.header.as_ref())?)
};
let proof = Decode::decode(&mut response.proof.as_ref())?;
let reply = self.checker.check_header_proof(&request, header, proof)?;
Ok(Reply::Header(reply))
} else {
Err(Error::UnexpectedResponse)
}
None => Err(Error::UnexpectedResponse)
},
None => Err(Error::UnexpectedResponse),
}
}
@@ -289,10 +281,10 @@ where
request: &Request<B>,
response: schema::v1::BlockResponse,
) -> Result<Reply<B>, Error> {
let request = if let Request::Body { request , .. } = &request {
let request = if let Request::Body { request, .. } = &request {
request
} else {
return Err(Error::UnexpectedResponse);
return Err(Error::UnexpectedResponse)
};
let body: Vec<_> = match response.blocks.into_iter().next() {
@@ -300,7 +292,8 @@ where
None => return Err(Error::UnexpectedResponse),
};
let body = body.into_iter()
let body = body
.into_iter()
.map(|extrinsic| B::Extrinsic::decode(&mut &extrinsic[..]))
.collect::<Result<_, _>>()?;
@@ -323,13 +316,14 @@ where
}
}
impl<B: Block> Stream for LightClientRequestSender<B> {
type Item = OutEvent;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
// If we have received responses to previously sent requests, check them and pass them on.
while let Poll::Ready(Some((sent_request, request_result))) = self.sent_requests.poll_next_unpin(cx) {
while let Poll::Ready(Some((sent_request, request_result))) =
self.sent_requests.poll_next_unpin(cx)
{
if let Some(info) = self.peers.get_mut(&sent_request.peer) {
if info.status != PeerStatus::Busy {
// If we get here, something is wrong with our internal handling of peer status
@@ -347,30 +341,38 @@ impl<B: Block> Stream for LightClientRequestSender<B> {
Err(oneshot::Canceled) => {
log::debug!("Oneshot for request to peer {} was canceled.", sent_request.peer);
self.remove_peer(sent_request.peer);
self.peerset.report_peer(sent_request.peer, ReputationChange::new_fatal("no response from peer"));
self.peerset.report_peer(
sent_request.peer,
ReputationChange::new_fatal("no response from peer"),
);
self.pending_requests.push_back(sent_request.into_pending());
continue;
}
continue
},
};
let decoded_request_result = request_result.map(|response| {
if sent_request.request.is_block_request() {
schema::v1::BlockResponse::decode(&response[..])
.map(|r| Response::Block(r))
schema::v1::BlockResponse::decode(&response[..]).map(|r| Response::Block(r))
} else {
schema::v1::light::Response::decode(&response[..])
.map(|r| Response::Light(r))
schema::v1::light::Response::decode(&response[..]).map(|r| Response::Light(r))
}
});
let response = match decoded_request_result {
Ok(Ok(response)) => response,
Ok(Err(e)) => {
log::debug!("Failed to decode response from peer {}: {:?}.", sent_request.peer, e);
log::debug!(
"Failed to decode response from peer {}: {:?}.",
sent_request.peer,
e
);
self.remove_peer(sent_request.peer);
self.peerset.report_peer(sent_request.peer, ReputationChange::new_fatal("invalid response from peer"));
self.peerset.report_peer(
sent_request.peer,
ReputationChange::new_fatal("invalid response from peer"),
);
self.pending_requests.push_back(sent_request.into_pending());
continue;
continue
},
Err(e) => {
log::debug!("Request to peer {} failed with {:?}.", sent_request.peer, e);
@@ -379,22 +381,19 @@ impl<B: Block> Stream for LightClientRequestSender<B> {
RequestFailure::NotConnected => {
self.remove_peer(sent_request.peer);
self.pending_requests.push_back(sent_request.into_pending());
}
},
RequestFailure::UnknownProtocol => {
debug_assert!(
false,
"Light client and block request protocol should be known when \
sending requests.",
);
}
},
RequestFailure::Refused => {
self.remove_peer(sent_request.peer);
self.peerset.report_peer(
sent_request.peer,
rep::REFUSED,
);
self.peerset.report_peer(sent_request.peer, rep::REFUSED);
self.pending_requests.push_back(sent_request.into_pending());
}
},
RequestFailure::Obsolete => {
debug_assert!(
false,
@@ -402,13 +401,10 @@ impl<B: Block> Stream for LightClientRequestSender<B> {
response receiver.",
);
self.pending_requests.push_back(sent_request.into_pending());
}
},
RequestFailure::Network(OutboundFailure::Timeout) => {
self.remove_peer(sent_request.peer);
self.peerset.report_peer(
sent_request.peer,
rep::TIMEOUT,
);
self.peerset.report_peer(sent_request.peer, rep::TIMEOUT);
self.pending_requests.push_back(sent_request.into_pending());
},
RequestFailure::Network(OutboundFailure::UnsupportedProtocols) => {
@@ -420,31 +416,27 @@ impl<B: Block> Stream for LightClientRequestSender<B> {
),
);
self.pending_requests.push_back(sent_request.into_pending());
}
},
RequestFailure::Network(OutboundFailure::DialFailure) => {
self.remove_peer(sent_request.peer);
self.peerset.report_peer(
sent_request.peer,
ReputationChange::new_fatal(
"failed to dial peer",
),
ReputationChange::new_fatal("failed to dial peer"),
);
self.pending_requests.push_back(sent_request.into_pending());
}
},
RequestFailure::Network(OutboundFailure::ConnectionClosed) => {
self.remove_peer(sent_request.peer);
self.peerset.report_peer(
sent_request.peer,
ReputationChange::new_fatal(
"connection to peer closed",
),
ReputationChange::new_fatal("connection to peer closed"),
);
self.pending_requests.push_back(sent_request.into_pending());
}
},
}
continue;
}
continue
},
};
match self.on_response(sent_request.peer, &sent_request.request, response) {
@@ -454,23 +446,23 @@ impl<B: Block> Stream for LightClientRequestSender<B> {
self.remove_peer(sent_request.peer);
self.peerset.report_peer(
sent_request.peer,
ReputationChange::new_fatal(
"unexpected response from peer",
),
ReputationChange::new_fatal("unexpected response from peer"),
);
self.pending_requests.push_back(sent_request.into_pending());
}
},
Err(other) => {
log::debug!("error handling response from peer {}: {}", sent_request.peer, other);
log::debug!(
"error handling response from peer {}: {}",
sent_request.peer,
other
);
self.remove_peer(sent_request.peer);
self.peerset.report_peer(
sent_request.peer,
ReputationChange::new_fatal(
"invalid response from peer",
),
ReputationChange::new_fatal("invalid response from peer"),
);
self.pending_requests.push_back(sent_request.into_pending())
}
},
}
}
@@ -497,7 +489,7 @@ impl<B: Block> Stream for LightClientRequestSender<B> {
peer = Some((*peer_id, peer_info));
break
},
_ => peer = Some((*peer_id, peer_info))
_ => peer = Some((*peer_id, peer_info)),
}
}
}
@@ -509,8 +501,8 @@ impl<B: Block> Stream for LightClientRequestSender<B> {
self.pending_requests.push_front(pending_request);
log::debug!("No peer available to send request to.");
break;
}
break
},
};
let request_bytes = match pending_request.request.serialize_request() {
@@ -519,7 +511,7 @@ impl<B: Block> Stream for LightClientRequestSender<B> {
log::debug!("failed to serialize request: {}", error);
pending_request.request.return_reply(Err(ClientError::RemoteFetchFailed));
continue
}
},
};
let (tx, rx) = oneshot::channel();
@@ -528,16 +520,15 @@ impl<B: Block> Stream for LightClientRequestSender<B> {
pending_request.attempts_left -= 1;
self.sent_requests.push(async move {
(pending_request.into_sent(peer_id), rx.await)
}.boxed());
self.sent_requests
.push(async move { (pending_request.into_sent(peer_id), rx.await) }.boxed());
return Poll::Ready(Some(OutEvent::SendRequest {
target: peer_id,
request: request_bytes,
pending_response: tx,
protocol_name: protocol,
}));
}))
}
Poll::Pending
@@ -557,7 +548,7 @@ pub enum OutEvent {
pending_response: oneshot::Sender<Result<Vec<u8>, RequestFailure>>,
/// The name of the protocol to use to send the request.
protocol_name: String,
}
},
}
/// Incoming response from remote.
@@ -592,7 +583,6 @@ enum Error {
}
/// The data to send back to the light client over the oneshot channel.
//
// It is unified here in order to be able to return it as a function
// result instead of delivering it to the client as a side effect of
// response processing.
@@ -605,7 +595,6 @@ enum Reply<B: Block> {
Extrinsics(Vec<B::Extrinsic>),
}
/// Information we have about some peer.
#[derive(Debug)]
struct PeerInfo<B: Block> {
@@ -615,10 +604,7 @@ struct PeerInfo<B: Block> {
impl<B: Block> Default for PeerInfo<B> {
fn default() -> Self {
PeerInfo {
best_block: None,
status: PeerStatus::Idle,
}
PeerInfo { best_block: None, status: PeerStatus::Idle }
}
}
@@ -635,7 +621,6 @@ enum PeerStatus {
///
/// The associated `oneshot::Sender` will be used to convey the result of
/// their request back to them (cf. `Reply`).
//
// This is modeled after light_dispatch.rs's `RequestData` which is not
// used because we currently only support a subset of those.
#[derive(Debug)]
@@ -645,43 +630,43 @@ pub enum Request<B: Block> {
/// Request.
request: RemoteBodyRequest<B::Header>,
/// [`oneshot::Sender`] to return response.
sender: oneshot::Sender<Result<Vec<B::Extrinsic>, ClientError>>
sender: oneshot::Sender<Result<Vec<B::Extrinsic>, ClientError>>,
},
/// Remote header request.
Header {
/// Request.
request: light::RemoteHeaderRequest<B::Header>,
/// [`oneshot::Sender`] to return response.
sender: oneshot::Sender<Result<B::Header, ClientError>>
sender: oneshot::Sender<Result<B::Header, ClientError>>,
},
/// Remote read request.
Read {
/// Request.
request: light::RemoteReadRequest<B::Header>,
/// [`oneshot::Sender`] to return response.
sender: oneshot::Sender<Result<HashMap<Vec<u8>, Option<Vec<u8>>>, ClientError>>
sender: oneshot::Sender<Result<HashMap<Vec<u8>, Option<Vec<u8>>>, ClientError>>,
},
/// Remote read child request.
ReadChild {
/// Request.
request: light::RemoteReadChildRequest<B::Header>,
/// [`oneshot::Sender`] to return response.
sender: oneshot::Sender<Result<HashMap<Vec<u8>, Option<Vec<u8>>>, ClientError>>
sender: oneshot::Sender<Result<HashMap<Vec<u8>, Option<Vec<u8>>>, ClientError>>,
},
/// Remote call request.
Call {
/// Request.
request: light::RemoteCallRequest<B::Header>,
/// [`oneshot::Sender`] to return response.
sender: oneshot::Sender<Result<Vec<u8>, ClientError>>
sender: oneshot::Sender<Result<Vec<u8>, ClientError>>,
},
/// Remote changes request.
Changes {
/// Request.
request: light::RemoteChangesRequest<B::Header>,
/// [`oneshot::Sender`] to return response.
sender: oneshot::Sender<Result<Vec<(NumberFor<B>, u32)>, ClientError>>
}
sender: oneshot::Sender<Result<Vec<(NumberFor<B>, u32)>, ClientError>>,
},
}
impl<B: Block> Request<B> {
@@ -728,19 +713,19 @@ impl<B: Block> Request<B> {
let mut buf = Vec::with_capacity(rq.encoded_len());
rq.encode(&mut buf)?;
return Ok(buf);
}
return Ok(buf)
},
Request::Header { request, .. } => {
let r = schema::v1::light::RemoteHeaderRequest { block: request.block.encode() };
schema::v1::light::request::Request::RemoteHeaderRequest(r)
}
},
Request::Read { request, .. } => {
let r = schema::v1::light::RemoteReadRequest {
block: request.block.encode(),
keys: request.keys.clone(),
};
schema::v1::light::request::Request::RemoteReadRequest(r)
}
},
Request::ReadChild { request, .. } => {
let r = schema::v1::light::RemoteReadChildRequest {
block: request.block.encode(),
@@ -748,7 +733,7 @@ impl<B: Block> Request<B> {
keys: request.keys.clone(),
};
schema::v1::light::request::Request::RemoteReadChildRequest(r)
}
},
Request::Call { request, .. } => {
let r = schema::v1::light::RemoteCallRequest {
block: request.block.encode(),
@@ -756,19 +741,22 @@ impl<B: Block> Request<B> {
data: request.call_data.clone(),
};
schema::v1::light::request::Request::RemoteCallRequest(r)
}
},
Request::Changes { request, .. } => {
let r = schema::v1::light::RemoteChangesRequest {
first: request.first_block.1.encode(),
last: request.last_block.1.encode(),
min: request.tries_roots.1.encode(),
max: request.max_block.1.encode(),
storage_key: request.storage_key.clone().map(|s| s.into_inner())
storage_key: request
.storage_key
.clone()
.map(|s| s.into_inner())
.unwrap_or_default(),
key: request.key.clone(),
};
schema::v1::light::request::Request::RemoteChangesRequest(r)
}
},
};
let rq = schema::v1::light::Request { request: Some(request) };
@@ -786,32 +774,35 @@ impl<B: Block> Request<B> {
Err(e) => send(Err(e), sender),
Ok(Reply::Extrinsics(x)) => send(Ok(x), sender),
reply => log::error!("invalid reply for body request: {:?}, {:?}", reply, request),
}
},
Request::Header { request, sender } => match result {
Err(e) => send(Err(e), sender),
Ok(Reply::Header(x)) => send(Ok(x), sender),
reply => log::error!("invalid reply for header request: {:?}, {:?}", reply, request),
}
reply =>
log::error!("invalid reply for header request: {:?}, {:?}", reply, request),
},
Request::Read { request, sender } => match result {
Err(e) => send(Err(e), sender),
Ok(Reply::MapVecU8OptVecU8(x)) => send(Ok(x), sender),
reply => log::error!("invalid reply for read request: {:?}, {:?}", reply, request),
}
},
Request::ReadChild { request, sender } => match result {
Err(e) => send(Err(e), sender),
Ok(Reply::MapVecU8OptVecU8(x)) => send(Ok(x), sender),
reply => log::error!("invalid reply for read child request: {:?}, {:?}", reply, request),
}
reply =>
log::error!("invalid reply for read child request: {:?}, {:?}", reply, request),
},
Request::Call { request, sender } => match result {
Err(e) => send(Err(e), sender),
Ok(Reply::VecU8(x)) => send(Ok(x), sender),
reply => log::error!("invalid reply for call request: {:?}, {:?}", reply, request),
}
},
Request::Changes { request, sender } => match result {
Err(e) => send(Err(e), sender),
Ok(Reply::VecNumberU32(x)) => send(Ok(x), sender),
reply => log::error!("invalid reply for changes request: {:?}, {:?}", reply, request),
}
reply =>
log::error!("invalid reply for changes request: {:?}, {:?}", reply, request),
},
}
}
}
@@ -819,19 +810,17 @@ impl<B: Block> Request<B> {
#[cfg(test)]
mod tests {
use super::*;
use crate::light_client_requests::tests::{DummyFetchChecker, protocol_id, peerset, dummy_header};
use crate::request_responses::OutboundFailure;
use crate::{
light_client_requests::tests::{dummy_header, peerset, protocol_id, DummyFetchChecker},
request_responses::OutboundFailure,
};
use assert_matches::assert_matches;
use futures::channel::oneshot;
use futures::executor::block_on;
use futures::poll;
use futures::{channel::oneshot, executor::block_on, poll};
use sc_client_api::StorageProof;
use sp_core::storage::ChildInfo;
use sp_runtime::generic::Header;
use sp_runtime::traits::BlakeTwo256;
use std::collections::HashSet;
use std::iter::FromIterator;
use sp_runtime::{generic::Header, traits::BlakeTwo256};
use std::{collections::HashSet, iter::FromIterator};
fn empty_proof() -> Vec<u8> {
StorageProof::empty().encode()
@@ -843,10 +832,7 @@ mod tests {
let (_peer_set, peer_set_handle) = peerset();
let mut sender = LightClientRequestSender::<Block>::new(
&protocol_id(),
Arc::new(DummyFetchChecker {
ok: true,
_mark: std::marker::PhantomData,
}),
Arc::new(DummyFetchChecker { ok: true, _mark: std::marker::PhantomData }),
peer_set_handle,
);
@@ -864,17 +850,15 @@ mod tests {
fn body_request_fields_encoded_properly() {
let (sender, _receiver) = oneshot::channel();
let request = Request::<Block>::Body {
request: RemoteBodyRequest {
header: dummy_header(),
retry_count: None,
},
request: RemoteBodyRequest { header: dummy_header(), retry_count: None },
sender,
};
let serialized_request = request.serialize_request().unwrap();
let deserialized_request = schema::v1::BlockRequest::decode(&serialized_request[..]).unwrap();
let deserialized_request =
schema::v1::BlockRequest::decode(&serialized_request[..]).unwrap();
assert!(BlockAttributes::from_be_u32(deserialized_request.fields)
.unwrap()
.contains(BlockAttributes::BODY));
.unwrap()
.contains(BlockAttributes::BODY));
}
#[test]
@@ -916,29 +900,26 @@ mod tests {
sender.request(Request::Call { request, sender: chan.0 }).unwrap();
assert_eq!(1, sender.pending_requests.len(), "Expect one pending request.");
let OutEvent::SendRequest { target, pending_response, .. } = block_on(sender.next()).unwrap();
assert!(
target == peer0 || target == peer1,
"Expect request to originate from known peer.",
);
let OutEvent::SendRequest { target, pending_response, .. } =
block_on(sender.next()).unwrap();
assert!(target == peer0 || target == peer1, "Expect request to originate from known peer.",);
// And we should have one busy peer.
assert!({
let (idle, busy): (Vec<_>, Vec<_>) = sender
.peers
.iter()
.partition(|(_, info)| info.status == PeerStatus::Idle);
idle.len() == 1
&& busy.len() == 1
&& (idle[0].0 == &peer0 || busy[0].0 == &peer0)
&& (idle[0].0 == &peer1 || busy[0].0 == &peer1)
let (idle, busy): (Vec<_>, Vec<_>) =
sender.peers.iter().partition(|(_, info)| info.status == PeerStatus::Idle);
idle.len() == 1 &&
busy.len() == 1 && (idle[0].0 == &peer0 || busy[0].0 == &peer0) &&
(idle[0].0 == &peer1 || busy[0].0 == &peer1)
});
assert_eq!(0, sender.pending_requests.len(), "Expect no pending request.");
assert_eq!(1, sender.sent_requests.len(), "Expect one request to be sent.");
// Report first attempt as timed out.
pending_response.send(Err(RequestFailure::Network(OutboundFailure::Timeout))).unwrap();
pending_response
.send(Err(RequestFailure::Network(OutboundFailure::Timeout)))
.unwrap();
// Expect a new request to be issued.
let OutEvent::SendRequest { pending_response, .. } = block_on(sender.next()).unwrap();
@@ -948,13 +929,17 @@ mod tests {
assert_eq!(1, sender.sent_requests.len(), "Expect new request to be issued.");
// Report second attempt as timed out.
pending_response.send(Err(RequestFailure::Network(OutboundFailure::Timeout))).unwrap();
pending_response
.send(Err(RequestFailure::Network(OutboundFailure::Timeout)))
.unwrap();
assert_matches!(
block_on(async { poll!(sender.next()) }), Poll::Pending,
block_on(async { poll!(sender.next()) }),
Poll::Pending,
"Expect sender to not issue another attempt.",
);
assert_matches!(
block_on(chan.1).unwrap(), Err(ClientError::RemoteFetchFailed),
block_on(chan.1).unwrap(),
Err(ClientError::RemoteFetchFailed),
"Expect request failure to be reported.",
);
assert_eq!(0, sender.peers.len(), "Expect no peer to be left");
@@ -988,12 +973,7 @@ mod tests {
call_data: vec![],
retry_count: Some(1),
};
sender
.request(Request::Call {
request,
sender: chan.0,
})
.unwrap();
sender.request(Request::Call { request, sender: chan.0 }).unwrap();
assert_eq!(1, sender.pending_requests.len(), "Expect one pending request.");
assert_eq!(0, sender.sent_requests.len(), "Expect zero sent requests.");
@@ -1003,9 +983,7 @@ mod tests {
assert_eq!(1, sender.sent_requests.len(), "Expect one sent request.");
let response = {
let r = schema::v1::light::RemoteCallResponse {
proof: empty_proof(),
};
let r = schema::v1::light::RemoteCallResponse { proof: empty_proof() };
let response = schema::v1::light::Response {
response: Some(schema::v1::light::response::Response::RemoteCallResponse(r)),
};
@@ -1017,7 +995,8 @@ mod tests {
pending_response.send(Ok(response)).unwrap();
assert_matches!(
block_on(async { poll!(sender.next()) }), Poll::Pending,
block_on(async { poll!(sender.next()) }),
Poll::Pending,
"Expect sender to not issue another attempt, given that there is no peer left.",
);
@@ -1050,12 +1029,7 @@ mod tests {
call_data: vec![],
retry_count: Some(1),
};
sender
.request(Request::Call {
request,
sender: chan.0,
})
.unwrap();
sender.request(Request::Call { request, sender: chan.0 }).unwrap();
assert_eq!(1, sender.pending_requests.len());
assert_eq!(0, sender.sent_requests.len());
@@ -1064,9 +1038,7 @@ mod tests {
assert_eq!(1, sender.sent_requests.len(), "Expect one sent request.");
let response = {
let r = schema::v1::light::RemoteReadResponse {
proof: empty_proof(),
}; // Not a RemoteCallResponse!
let r = schema::v1::light::RemoteReadResponse { proof: empty_proof() }; // Not a RemoteCallResponse!
let response = schema::v1::light::Response {
response: Some(schema::v1::light::response::Response::RemoteReadResponse(r)),
};
@@ -1077,7 +1049,8 @@ mod tests {
pending_response.send(Ok(response)).unwrap();
assert_matches!(
block_on(async { poll!(sender.next()) }), Poll::Pending,
block_on(async { poll!(sender.next()) }),
Poll::Pending,
"Expect sender to not issue another attempt, given that there is no peer left.",
);
@@ -1114,12 +1087,7 @@ mod tests {
call_data: vec![],
retry_count: Some(3), // Attempt up to three retries.
};
sender
.request(Request::Call {
request,
sender: chan.0,
})
.unwrap();
sender.request(Request::Call { request, sender: chan.0 }).unwrap();
assert_eq!(1, sender.pending_requests.len());
assert_eq!(0, sender.sent_requests.len());
@@ -1132,9 +1100,7 @@ mod tests {
for (i, _peer) in peers.iter().enumerate() {
// Construct an invalid response
let response = {
let r = schema::v1::light::RemoteCallResponse {
proof: empty_proof(),
};
let r = schema::v1::light::RemoteCallResponse { proof: empty_proof() };
let response = schema::v1::light::Response {
response: Some(schema::v1::light::response::Response::RemoteCallResponse(r)),
};
@@ -1152,13 +1118,11 @@ mod tests {
} else {
// Last peer and last attempt.
assert_matches!(
block_on(async { poll!(sender.next()) }), Poll::Pending,
block_on(async { poll!(sender.next()) }),
Poll::Pending,
"Expect sender to not issue another attempt, given that there is no peer left.",
);
assert_matches!(
chan.1.try_recv(),
Ok(Some(Err(ClientError::RemoteFetchFailed)))
)
assert_matches!(chan.1.try_recv(), Ok(Some(Err(ClientError::RemoteFetchFailed))))
}
}
}
@@ -1187,35 +1151,27 @@ mod tests {
proof: empty_proof(),
};
schema::v1::light::Response {
response: Some(schema::v1::light::response::Response::RemoteHeaderResponse(
r,
)),
response: Some(schema::v1::light::response::Response::RemoteHeaderResponse(r)),
}
}
},
Request::Read { .. } => {
let r = schema::v1::light::RemoteReadResponse {
proof: empty_proof(),
};
let r = schema::v1::light::RemoteReadResponse { proof: empty_proof() };
schema::v1::light::Response {
response: Some(schema::v1::light::response::Response::RemoteReadResponse(r)),
}
}
},
Request::ReadChild { .. } => {
let r = schema::v1::light::RemoteReadResponse {
proof: empty_proof(),
};
let r = schema::v1::light::RemoteReadResponse { proof: empty_proof() };
schema::v1::light::Response {
response: Some(schema::v1::light::response::Response::RemoteReadResponse(r)),
}
}
},
Request::Call { .. } => {
let r = schema::v1::light::RemoteCallResponse {
proof: empty_proof(),
};
let r = schema::v1::light::RemoteCallResponse { proof: empty_proof() };
schema::v1::light::Response {
response: Some(schema::v1::light::response::Response::RemoteCallResponse(r)),
}
}
},
Request::Changes { .. } => {
let r = schema::v1::light::RemoteChangesResponse {
max: std::iter::repeat(1).take(32).collect(),
@@ -1226,7 +1182,7 @@ mod tests {
schema::v1::light::Response {
response: Some(schema::v1::light::response::Response::RemoteChangesResponse(r)),
}
}
},
};
let response = {
@@ -1245,7 +1201,8 @@ mod tests {
pending_response.send(Ok(response)).unwrap();
assert_matches!(
block_on(async { poll!(sender.next()) }), Poll::Pending,
block_on(async { poll!(sender.next()) }),
Poll::Pending,
"Expect sender to not issue another attempt, given that there is no peer left.",
);
@@ -1263,10 +1220,7 @@ mod tests {
call_data: vec![],
retry_count: None,
};
issue_request(Request::Call {
request,
sender: chan.0,
});
issue_request(Request::Call { request, sender: chan.0 });
assert_matches!(chan.1.try_recv(), Ok(Some(Ok(_))))
}
@@ -1279,10 +1233,7 @@ mod tests {
keys: vec![b":key".to_vec()],
retry_count: None,
};
issue_request(Request::Read {
request,
sender: chan.0,
});
issue_request(Request::Read { request, sender: chan.0 });
assert_matches!(chan.1.try_recv(), Ok(Some(Ok(_))))
}
@@ -1297,10 +1248,7 @@ mod tests {
keys: vec![b":key".to_vec()],
retry_count: None,
};
issue_request(Request::ReadChild {
request,
sender: chan.0,
});
issue_request(Request::ReadChild { request, sender: chan.0 });
assert_matches!(chan.1.try_recv(), Ok(Some(Ok(_))))
}
@@ -1312,10 +1260,7 @@ mod tests {
block: 1,
retry_count: None,
};
issue_request(Request::Header {
request,
sender: chan.0,
});
issue_request(Request::Header { request, sender: chan.0 });
assert_matches!(chan.1.try_recv(), Ok(Some(Ok(_))))
}
@@ -1336,10 +1281,7 @@ mod tests {
storage_key: None,
retry_count: None,
};
issue_request(Request::Changes {
request,
sender: chan.0,
});
issue_request(Request::Changes { request, sender: chan.0 });
assert_matches!(chan.1.try_recv(), Ok(Some(Ok(_))))
}
}
@@ -22,7 +22,10 @@
use libp2p::{core::ConnectedPoint, Multiaddr};
use serde::{Deserialize, Serialize};
use std::{collections::{HashMap, HashSet}, time::Duration};
use std::{
collections::{HashMap, HashSet},
time::Duration,
};
/// Returns general information about the networking.
///
@@ -90,13 +93,9 @@ pub enum PeerEndpoint {
impl From<ConnectedPoint> for PeerEndpoint {
fn from(endpoint: ConnectedPoint) -> Self {
match endpoint {
ConnectedPoint::Dialer { address } =>
PeerEndpoint::Dialing(address),
ConnectedPoint::Dialer { address } => PeerEndpoint::Dialing(address),
ConnectedPoint::Listener { local_addr, send_back_addr } =>
PeerEndpoint::Listening {
local_addr,
send_back_addr
}
PeerEndpoint::Listening { local_addr, send_back_addr },
}
}
}
+19 -17
View File
@@ -23,13 +23,19 @@ use crate::light_client_requests;
use futures::{channel::oneshot, prelude::*};
use parking_lot::Mutex;
use sc_client_api::{
FetchChecker, Fetcher, RemoteBodyRequest, RemoteCallRequest, RemoteChangesRequest,
RemoteHeaderRequest, RemoteReadChildRequest, RemoteReadRequest, StorageProof, ChangesProof,
ChangesProof, FetchChecker, Fetcher, RemoteBodyRequest, RemoteCallRequest,
RemoteChangesRequest, RemoteHeaderRequest, RemoteReadChildRequest, RemoteReadRequest,
StorageProof,
};
use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender};
use sp_blockchain::Error as ClientError;
use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor};
use std::{collections::HashMap, pin::Pin, sync::Arc, task::Context, task::Poll};
use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender};
use std::{
collections::HashMap,
pin::Pin,
sync::Arc,
task::{Context, Poll},
};
/// Implements the `Fetcher` trait of the client. Makes it possible for the light client to perform
/// network requests for some state.
@@ -45,13 +51,13 @@ pub struct OnDemand<B: BlockT> {
/// Note that a better alternative would be to use a MPMC queue here, and add a `poll` method
/// from the `OnDemand`. However there exists no popular implementation of MPMC channels in
/// asynchronous Rust at the moment
requests_queue: Mutex<Option<TracingUnboundedReceiver<light_client_requests::sender::Request<B>>>>,
requests_queue:
Mutex<Option<TracingUnboundedReceiver<light_client_requests::sender::Request<B>>>>,
/// Sending side of `requests_queue`.
requests_send: TracingUnboundedSender<light_client_requests::sender::Request<B>>,
}
#[derive(Debug, thiserror::Error)]
#[error("AlwaysBadChecker")]
struct ErrorAlwaysBadChecker;
@@ -83,7 +89,7 @@ impl<Block: BlockT> FetchChecker<Block> for AlwaysBadChecker {
&self,
_request: &RemoteReadRequest<Block::Header>,
_remote_proof: StorageProof,
) -> Result<HashMap<Vec<u8>,Option<Vec<u8>>>, ClientError> {
) -> Result<HashMap<Vec<u8>, Option<Vec<u8>>>, ClientError> {
Err(ErrorAlwaysBadChecker.into())
}
@@ -106,7 +112,7 @@ impl<Block: BlockT> FetchChecker<Block> for AlwaysBadChecker {
fn check_changes_proof(
&self,
_request: &RemoteChangesRequest<Block::Header>,
_remote_proof: ChangesProof<Block::Header>
_remote_proof: ChangesProof<Block::Header>,
) -> Result<Vec<(NumberFor<Block>, u32)>, ClientError> {
Err(ErrorAlwaysBadChecker.into())
}
@@ -114,7 +120,7 @@ impl<Block: BlockT> FetchChecker<Block> for AlwaysBadChecker {
fn check_body_proof(
&self,
_request: &RemoteBodyRequest<Block::Header>,
_body: Vec<Block::Extrinsic>
_body: Vec<Block::Extrinsic>,
) -> Result<Vec<Block::Extrinsic>, ClientError> {
Err(ErrorAlwaysBadChecker.into())
}
@@ -129,11 +135,7 @@ where
let (requests_send, requests_queue) = tracing_unbounded("mpsc_ondemand");
let requests_queue = Mutex::new(Some(requests_queue));
OnDemand {
checker,
requests_queue,
requests_send,
}
OnDemand { checker, requests_queue, requests_send }
}
/// Get checker reference.
@@ -148,9 +150,9 @@ where
///
/// If this function returns `None`, that means that the receiver has already been extracted in
/// the past, and therefore that something already handles the requests.
pub(crate) fn extract_receiver(&self)
-> Option<TracingUnboundedReceiver<light_client_requests::sender::Request<B>>>
{
pub(crate) fn extract_receiver(
&self,
) -> Option<TracingUnboundedReceiver<light_client_requests::sender::Request<B>>> {
self.requests_queue.lock().take()
}
}
+68 -48
View File
@@ -16,24 +16,33 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::utils::interval;
use fnv::FnvHashMap;
use futures::prelude::*;
use libp2p::Multiaddr;
use libp2p::core::connection::{ConnectionId, ListenerId};
use libp2p::core::{ConnectedPoint, either::EitherOutput, PeerId, PublicKey};
use libp2p::swarm::{IntoProtocolsHandler, IntoProtocolsHandlerSelect, ProtocolsHandler};
use libp2p::swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters};
use libp2p::identify::{Identify, IdentifyConfig, IdentifyEvent, IdentifyInfo};
use libp2p::ping::{Ping, PingConfig, PingEvent, PingSuccess};
use log::{debug, trace, error};
use libp2p::{
core::{
connection::{ConnectionId, ListenerId},
either::EitherOutput,
ConnectedPoint, PeerId, PublicKey,
},
identify::{Identify, IdentifyConfig, IdentifyEvent, IdentifyInfo},
ping::{Ping, PingConfig, PingEvent, PingSuccess},
swarm::{
IntoProtocolsHandler, IntoProtocolsHandlerSelect, NetworkBehaviour, NetworkBehaviourAction,
PollParameters, ProtocolsHandler,
},
Multiaddr,
};
use log::{debug, error, trace};
use smallvec::SmallVec;
use std::{error, io};
use std::collections::hash_map::Entry;
use std::pin::Pin;
use std::task::{Context, Poll};
use std::time::Duration;
use std::{
collections::hash_map::Entry,
error, io,
pin::Pin,
task::{Context, Poll},
time::Duration,
};
use wasm_timer::Instant;
use crate::utils::interval;
/// Time after we disconnect from a node before we purge its information from the cache.
const CACHE_EXPIRE: Duration = Duration::from_secs(10 * 60);
@@ -70,21 +79,13 @@ impl NodeInfo {
fn new(endpoint: ConnectedPoint) -> Self {
let mut endpoints = SmallVec::new();
endpoints.push(endpoint);
NodeInfo {
info_expire: None,
endpoints,
client_version: None,
latest_ping: None,
}
NodeInfo { info_expire: None, endpoints, client_version: None, latest_ping: None }
}
}
impl PeerInfoBehaviour {
/// Builds a new `PeerInfoBehaviour`.
pub fn new(
user_agent: String,
local_public_key: PublicKey,
) -> Self {
pub fn new(user_agent: String, local_public_key: PublicKey) -> Self {
let identify = {
let cfg = IdentifyConfig::new("/substrate/1.0".to_string(), local_public_key)
.with_agent_version(user_agent);
@@ -172,7 +173,7 @@ pub enum PeerInfoEvent {
impl NetworkBehaviour for PeerInfoBehaviour {
type ProtocolsHandler = IntoProtocolsHandlerSelect<
<Ping as NetworkBehaviour>::ProtocolsHandler,
<Identify as NetworkBehaviour>::ProtocolsHandler
<Identify as NetworkBehaviour>::ProtocolsHandler,
>;
type OutEvent = PeerInfoEvent;
@@ -191,13 +192,18 @@ impl NetworkBehaviour for PeerInfoBehaviour {
self.identify.inject_connected(peer_id);
}
fn inject_connection_established(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) {
fn inject_connection_established(
&mut self,
peer_id: &PeerId,
conn: &ConnectionId,
endpoint: &ConnectedPoint,
) {
self.ping.inject_connection_established(peer_id, conn, endpoint);
self.identify.inject_connection_established(peer_id, conn, endpoint);
match self.nodes_info.entry(peer_id.clone()) {
Entry::Vacant(e) => {
e.insert(NodeInfo::new(endpoint.clone()));
}
},
Entry::Occupied(e) => {
let e = e.into_mut();
if e.info_expire.as_ref().map(|exp| *exp < Instant::now()).unwrap_or(false) {
@@ -206,11 +212,16 @@ impl NetworkBehaviour for PeerInfoBehaviour {
}
e.info_expire = None;
e.endpoints.push(endpoint.clone());
}
},
}
}
fn inject_connection_closed(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) {
fn inject_connection_closed(
&mut self,
peer_id: &PeerId,
conn: &ConnectionId,
endpoint: &ConnectedPoint,
) {
self.ping.inject_connection_closed(peer_id, conn, endpoint);
self.identify.inject_connection_closed(peer_id, conn, endpoint);
@@ -238,7 +249,7 @@ impl NetworkBehaviour for PeerInfoBehaviour {
&mut self,
peer_id: PeerId,
connection: ConnectionId,
event: <<Self::ProtocolsHandler as IntoProtocolsHandler>::Handler as ProtocolsHandler>::OutEvent
event: <<Self::ProtocolsHandler as IntoProtocolsHandler>::Handler as ProtocolsHandler>::OutEvent,
) {
match event {
EitherOutput::First(event) => self.ping.inject_event(peer_id, connection, event),
@@ -246,7 +257,12 @@ impl NetworkBehaviour for PeerInfoBehaviour {
}
}
fn inject_addr_reach_failure(&mut self, peer_id: Option<&PeerId>, addr: &Multiaddr, error: &dyn std::error::Error) {
fn inject_addr_reach_failure(
&mut self,
peer_id: Option<&PeerId>,
addr: &Multiaddr,
error: &dyn std::error::Error,
) {
self.ping.inject_addr_reach_failure(peer_id, addr, error);
self.identify.inject_addr_reach_failure(peer_id, addr, error);
}
@@ -300,7 +316,7 @@ impl NetworkBehaviour for PeerInfoBehaviour {
<<Self::ProtocolsHandler as IntoProtocolsHandler>::Handler as ProtocolsHandler>::InEvent,
Self::OutEvent
>
> {
>{
loop {
match self.ping.poll(cx, params) {
Poll::Pending => break,
@@ -317,28 +333,29 @@ impl NetworkBehaviour for PeerInfoBehaviour {
return Poll::Ready(NetworkBehaviourAction::NotifyHandler {
peer_id,
handler,
event: EitherOutput::First(event)
event: EitherOutput::First(event),
}),
Poll::Ready(NetworkBehaviourAction::ReportObservedAddr { address, score }) =>
return Poll::Ready(NetworkBehaviourAction::ReportObservedAddr { address, score }),
return Poll::Ready(NetworkBehaviourAction::ReportObservedAddr {
address,
score,
}),
}
}
loop {
match self.identify.poll(cx, params) {
Poll::Pending => break,
Poll::Ready(NetworkBehaviourAction::GenerateEvent(event)) => {
match event {
IdentifyEvent::Received { peer_id, info, .. } => {
self.handle_identify_report(&peer_id, &info);
let event = PeerInfoEvent::Identified { peer_id, info };
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(event));
}
IdentifyEvent::Error { peer_id, error } =>
debug!(target: "sub-libp2p", "Identification with peer {:?} failed => {}", peer_id, error),
IdentifyEvent::Pushed { .. } => {}
IdentifyEvent::Sent { .. } => {}
}
Poll::Ready(NetworkBehaviourAction::GenerateEvent(event)) => match event {
IdentifyEvent::Received { peer_id, info, .. } => {
self.handle_identify_report(&peer_id, &info);
let event = PeerInfoEvent::Identified { peer_id, info };
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(event))
},
IdentifyEvent::Error { peer_id, error } =>
debug!(target: "sub-libp2p", "Identification with peer {:?} failed => {}", peer_id, error),
IdentifyEvent::Pushed { .. } => {},
IdentifyEvent::Sent { .. } => {},
},
Poll::Ready(NetworkBehaviourAction::DialAddress { address }) =>
return Poll::Ready(NetworkBehaviourAction::DialAddress { address }),
@@ -348,10 +365,13 @@ impl NetworkBehaviour for PeerInfoBehaviour {
return Poll::Ready(NetworkBehaviourAction::NotifyHandler {
peer_id,
handler,
event: EitherOutput::Second(event)
event: EitherOutput::Second(event),
}),
Poll::Ready(NetworkBehaviourAction::ReportObservedAddr { address, score }) =>
return Poll::Ready(NetworkBehaviourAction::ReportObservedAddr { address, score }),
return Poll::Ready(NetworkBehaviourAction::ReportObservedAddr {
address,
score,
}),
}
}
File diff suppressed because it is too large Load Diff
@@ -20,8 +20,7 @@
//! events that happen on the network like DHT get/put results received.
use bytes::Bytes;
use libp2p::core::PeerId;
use libp2p::kad::record::Key;
use libp2p::{core::PeerId, kad::record::Key};
use std::borrow::Cow;
/// Events generated by DHT as a response to get_value and put_value requests.
@@ -18,16 +18,17 @@
//! Network packet message types. These get serialized and put into the lower level protocol payload.
use bitflags::bitflags;
use sp_runtime::{ConsensusEngineId, traits::{Block as BlockT, Header as HeaderT}};
use codec::{Encode, Decode, Input, Output, Error};
pub use self::generic::{
BlockAnnounce, RemoteCallRequest, RemoteReadRequest,
RemoteHeaderRequest, RemoteHeaderResponse,
RemoteChangesRequest, RemoteChangesResponse,
FromBlock, RemoteReadChildRequest, Roles,
BlockAnnounce, FromBlock, RemoteCallRequest, RemoteChangesRequest, RemoteChangesResponse,
RemoteHeaderRequest, RemoteHeaderResponse, RemoteReadChildRequest, RemoteReadRequest, Roles,
};
use bitflags::bitflags;
use codec::{Decode, Encode, Error, Input, Output};
use sc_client_api::StorageProof;
use sp_runtime::{
traits::{Block as BlockT, Header as HeaderT},
ConsensusEngineId,
};
/// A unique ID of a request.
pub type RequestId = u64;
@@ -41,24 +42,16 @@ pub type Message<B> = generic::Message<
>;
/// Type alias for using the block request type using block type parameters.
pub type BlockRequest<B> = generic::BlockRequest<
<B as BlockT>::Hash,
<<B as BlockT>::Header as HeaderT>::Number,
>;
pub type BlockRequest<B> =
generic::BlockRequest<<B as BlockT>::Hash, <<B as BlockT>::Header as HeaderT>::Number>;
/// Type alias for using the BlockData type using block type parameters.
pub type BlockData<B> = generic::BlockData<
<B as BlockT>::Header,
<B as BlockT>::Hash,
<B as BlockT>::Extrinsic,
>;
pub type BlockData<B> =
generic::BlockData<<B as BlockT>::Header, <B as BlockT>::Hash, <B as BlockT>::Extrinsic>;
/// Type alias for using the BlockResponse type using block type parameters.
pub type BlockResponse<B> = generic::BlockResponse<
<B as BlockT>::Header,
<B as BlockT>::Hash,
<B as BlockT>::Extrinsic,
>;
pub type BlockResponse<B> =
generic::BlockResponse<<B as BlockT>::Header, <B as BlockT>::Hash, <B as BlockT>::Extrinsic>;
/// A set of transactions.
pub type Transactions<E> = Vec<E>;
@@ -168,14 +161,13 @@ impl<H: HeaderT> generic::BlockAnnounce<H> {
/// Generic types.
pub mod generic {
use bitflags::bitflags;
use codec::{Encode, Decode, Input, Output};
use sp_runtime::{EncodedJustification, Justifications};
use super::{
RemoteReadResponse, Transactions, Direction,
RequestId, BlockAttributes, RemoteCallResponse, ConsensusEngineId,
BlockState, StorageProof,
BlockAttributes, BlockState, ConsensusEngineId, Direction, RemoteCallResponse,
RemoteReadResponse, RequestId, StorageProof, Transactions,
};
use bitflags::bitflags;
use codec::{Decode, Encode, Input, Output};
use sp_runtime::{EncodedJustification, Justifications};
bitflags! {
/// Bitmask of the roles that a node fulfills.
@@ -358,11 +350,12 @@ pub mod generic {
let compact = CompactStatus::decode(value)?;
let chain_status = match <Vec<u8>>::decode(value) {
Ok(v) => v,
Err(e) => if compact.version <= LAST_CHAIN_STATUS_VERSION {
return Err(e)
} else {
Vec::new()
}
Err(e) =>
if compact.version <= LAST_CHAIN_STATUS_VERSION {
return Err(e)
} else {
Vec::new()
},
};
let CompactStatus {
@@ -443,11 +436,7 @@ pub mod generic {
let header = H::decode(input)?;
let state = BlockState::decode(input).ok();
let data = Vec::decode(input).ok();
Ok(BlockAnnounce {
header,
state,
data,
})
Ok(BlockAnnounce { header, state, data })
}
}
@@ -19,10 +19,12 @@
//! Implementation of libp2p's `NetworkBehaviour` trait that establishes communications and opens
//! notifications substreams.
pub use self::behaviour::{Notifications, NotificationsOut, ProtocolConfig};
pub use self::handler::{NotifsHandlerError, NotificationsSink, Ready};
pub use self::{
behaviour::{Notifications, NotificationsOut, ProtocolConfig},
handler::{NotificationsSink, NotifsHandlerError, Ready},
};
mod behaviour;
mod handler;
mod upgrade;
mod tests;
mod upgrade;
File diff suppressed because it is too large Load Diff
@@ -57,31 +57,39 @@
//! It is illegal to send a [`NotifsHandlerIn::Open`] before a previously-emitted
//! [`NotifsHandlerIn::Open`] has gotten an answer.
use crate::protocol::notifications::{
upgrade::{
NotificationsIn, NotificationsOut, NotificationsInSubstream, NotificationsOutSubstream,
NotificationsHandshakeError, UpgradeCollec
},
use crate::protocol::notifications::upgrade::{
NotificationsHandshakeError, NotificationsIn, NotificationsInSubstream, NotificationsOut,
NotificationsOutSubstream, UpgradeCollec,
};
use bytes::BytesMut;
use libp2p::core::{ConnectedPoint, PeerId, upgrade::{InboundUpgrade, OutboundUpgrade}};
use libp2p::swarm::{
ProtocolsHandler, ProtocolsHandlerEvent,
IntoProtocolsHandler,
KeepAlive,
ProtocolsHandlerUpgrErr,
SubstreamProtocol,
NegotiatedSubstream,
};
use futures::{
channel::mpsc,
lock::{Mutex as FuturesMutex, MutexGuard as FuturesMutexGuard},
prelude::*
prelude::*,
};
use libp2p::{
core::{
upgrade::{InboundUpgrade, OutboundUpgrade},
ConnectedPoint, PeerId,
},
swarm::{
IntoProtocolsHandler, KeepAlive, NegotiatedSubstream, ProtocolsHandler,
ProtocolsHandlerEvent, ProtocolsHandlerUpgrErr, SubstreamProtocol,
},
};
use log::error;
use parking_lot::{Mutex, RwLock};
use std::{borrow::Cow, collections::VecDeque, mem, pin::Pin, str, sync::Arc, task::{Context, Poll}, time::Duration};
use std::{
borrow::Cow,
collections::VecDeque,
mem,
pin::Pin,
str,
sync::Arc,
task::{Context, Poll},
time::Duration,
};
use wasm_timer::Instant;
/// Number of pending notifications in asynchronous contexts.
@@ -131,7 +139,7 @@ pub struct NotifsHandler {
/// Events to return in priority from `poll`.
events_queue: VecDeque<
ProtocolsHandlerEvent<NotificationsOut, usize, NotifsHandlerOut, NotifsHandlerError>
ProtocolsHandlerEvent<NotificationsOut, usize, NotifsHandlerOut, NotifsHandlerError>,
>,
}
@@ -195,10 +203,12 @@ enum State {
/// We use two different channels in order to have two different channel sizes, but from
/// the receiving point of view, the two channels are the same.
/// The receivers are fused in case the user drops the [`NotificationsSink`] entirely.
notifications_sink_rx: stream::Peekable<stream::Select<
stream::Fuse<mpsc::Receiver<NotificationsSinkMessage>>,
stream::Fuse<mpsc::Receiver<NotificationsSinkMessage>>
>>,
notifications_sink_rx: stream::Peekable<
stream::Select<
stream::Fuse<mpsc::Receiver<NotificationsSinkMessage>>,
stream::Fuse<mpsc::Receiver<NotificationsSinkMessage>>,
>,
>,
/// Outbound substream that has been accepted by the remote.
///
@@ -220,28 +230,33 @@ impl IntoProtocolsHandler for NotifsHandlerProto {
type Handler = NotifsHandler;
fn inbound_protocol(&self) -> UpgradeCollec<NotificationsIn> {
self.protocols.iter()
.map(|cfg| NotificationsIn::new(cfg.name.clone(), cfg.fallback_names.clone(), cfg.max_notification_size))
self.protocols
.iter()
.map(|cfg| {
NotificationsIn::new(
cfg.name.clone(),
cfg.fallback_names.clone(),
cfg.max_notification_size,
)
})
.collect::<UpgradeCollec<_>>()
}
fn into_handler(self, peer_id: &PeerId, connected_point: &ConnectedPoint) -> Self::Handler {
NotifsHandler {
protocols: self.protocols.into_iter().map(|config| {
let in_upgrade = NotificationsIn::new(
config.name.clone(),
config.fallback_names.clone(),
config.max_notification_size
);
protocols: self
.protocols
.into_iter()
.map(|config| {
let in_upgrade = NotificationsIn::new(
config.name.clone(),
config.fallback_names.clone(),
config.max_notification_size,
);
Protocol {
config,
in_upgrade,
state: State::Closed {
pending_opening: false,
},
}
}).collect(),
Protocol { config, in_upgrade, state: State::Closed { pending_opening: false } }
})
.collect(),
peer_id: peer_id.clone(),
endpoint: connected_point.clone(),
when_connection_open: Instant::now(),
@@ -363,9 +378,7 @@ struct NotificationsSinkInner {
enum NotificationsSinkMessage {
/// Message emitted by [`NotificationsSink::reserve_notification`] and
/// [`NotificationsSink::write_notification_now`].
Notification {
message: Vec<u8>,
},
Notification { message: Vec<u8> },
/// Must close the connection.
ForceClose,
@@ -386,14 +399,10 @@ impl NotificationsSink {
/// error to send a notification using an unknown protocol.
///
/// This method will be removed in a future version.
pub fn send_sync_notification<'a>(
&'a self,
message: impl Into<Vec<u8>>
) {
pub fn send_sync_notification<'a>(&'a self, message: impl Into<Vec<u8>>) {
let mut lock = self.inner.sync_channel.lock();
let result = lock.try_send(NotificationsSinkMessage::Notification {
message: message.into()
});
let result =
lock.try_send(NotificationsSinkMessage::Notification { message: message.into() });
if result.is_err() {
// Cloning the `mpsc::Sender` guarantees the allocation of an extra spot in the
@@ -433,13 +442,10 @@ impl<'a> Ready<'a> {
/// Consumes this slots reservation and actually queues the notification.
///
/// Returns an error if the substream has been closed.
pub fn send(
mut self,
notification: impl Into<Vec<u8>>
) -> Result<(), ()> {
self.lock.start_send(NotificationsSinkMessage::Notification {
message: notification.into(),
}).map_err(|_| ())
pub fn send(mut self, notification: impl Into<Vec<u8>>) -> Result<(), ()> {
self.lock
.start_send(NotificationsSinkMessage::Notification { message: notification.into() })
.map_err(|_| ())
}
}
@@ -457,12 +463,8 @@ impl NotifsHandlerProto {
/// handshake, and the maximum allowed size of a notification. At the moment, the message
/// is always the same whether we open a substream ourselves or respond to handshake from
/// the remote.
pub fn new(
list: impl Into<Vec<ProtocolConfig>>,
) -> Self {
NotifsHandlerProto {
protocols: list.into(),
}
pub fn new(list: impl Into<Vec<ProtocolConfig>>) -> Self {
NotifsHandlerProto { protocols: list.into() }
}
}
@@ -477,7 +479,9 @@ impl ProtocolsHandler for NotifsHandler {
type InboundOpenInfo = ();
fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol, ()> {
let protocols = self.protocols.iter()
let protocols = self
.protocols
.iter()
.map(|p| p.in_upgrade.clone())
.collect::<UpgradeCollec<_>>();
@@ -486,17 +490,16 @@ impl ProtocolsHandler for NotifsHandler {
fn inject_fully_negotiated_inbound(
&mut self,
(mut in_substream_open, protocol_index):
<Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output,
(): ()
(mut in_substream_open, protocol_index): <Self::InboundProtocol as InboundUpgrade<
NegotiatedSubstream,
>>::Output,
(): (),
) {
let mut protocol_info = &mut self.protocols[protocol_index];
match protocol_info.state {
State::Closed { pending_opening } => {
self.events_queue.push_back(ProtocolsHandlerEvent::Custom(
NotifsHandlerOut::OpenDesiredByRemote {
protocol_index,
}
NotifsHandlerOut::OpenDesiredByRemote { protocol_index },
));
protocol_info.state = State::OpenDesiredByRemote {
@@ -512,13 +515,13 @@ impl ProtocolsHandler for NotifsHandler {
// in mind that it is invalid for the remote to open multiple such
// substreams, and therefore sending a "RST" is the most correct thing
// to do.
return;
return
},
State::Opening { ref mut in_substream, .. } |
State::Open { ref mut in_substream, .. } => {
if in_substream.is_some() {
// Same remark as above.
return;
return
}
// Create `handshake_message` on a separate line to be sure that the
@@ -533,18 +536,18 @@ impl ProtocolsHandler for NotifsHandler {
fn inject_fully_negotiated_outbound(
&mut self,
new_open: <Self::OutboundProtocol as OutboundUpgrade<NegotiatedSubstream>>::Output,
protocol_index: Self::OutboundOpenInfo
protocol_index: Self::OutboundOpenInfo,
) {
match self.protocols[protocol_index].state {
State::Closed { ref mut pending_opening } |
State::OpenDesiredByRemote { ref mut pending_opening, .. } => {
debug_assert!(*pending_opening);
*pending_opening = false;
}
},
State::Open { .. } => {
error!(target: "sub-libp2p", "☎️ State mismatch in notifications handler");
debug_assert!(false);
}
},
State::Opening { ref mut in_substream } => {
let (async_tx, async_rx) = mpsc::channel(ASYNC_NOTIFICATIONS_BUFFER_SIZE);
let (sync_tx, sync_rx) = mpsc::channel(SYNC_NOTIFICATIONS_BUFFER_SIZE);
@@ -557,7 +560,8 @@ impl ProtocolsHandler for NotifsHandler {
};
self.protocols[protocol_index].state = State::Open {
notifications_sink_rx: stream::select(async_rx.fuse(), sync_rx.fuse()).peekable(),
notifications_sink_rx: stream::select(async_rx.fuse(), sync_rx.fuse())
.peekable(),
out_substream: Some(new_open.substream),
in_substream: in_substream.take(),
};
@@ -568,10 +572,10 @@ impl ProtocolsHandler for NotifsHandler {
negotiated_fallback: new_open.negotiated_fallback,
endpoint: self.endpoint.clone(),
received_handshake: new_open.handshake,
notifications_sink
}
notifications_sink,
},
));
}
},
}
}
@@ -586,18 +590,18 @@ impl ProtocolsHandler for NotifsHandler {
protocol_info.config.name.clone(),
protocol_info.config.fallback_names.clone(),
protocol_info.config.handshake.read().clone(),
protocol_info.config.max_notification_size
protocol_info.config.max_notification_size,
);
self.events_queue.push_back(ProtocolsHandlerEvent::OutboundSubstreamRequest {
protocol: SubstreamProtocol::new(proto, protocol_index)
.with_timeout(OPEN_TIMEOUT),
});
self.events_queue.push_back(
ProtocolsHandlerEvent::OutboundSubstreamRequest {
protocol: SubstreamProtocol::new(proto, protocol_index)
.with_timeout(OPEN_TIMEOUT),
},
);
}
protocol_info.state = State::Opening {
in_substream: None,
};
protocol_info.state = State::Opening { in_substream: None };
},
State::OpenDesiredByRemote { pending_opening, in_substream } => {
let handshake_message = protocol_info.config.handshake.read().clone();
@@ -610,27 +614,27 @@ impl ProtocolsHandler for NotifsHandler {
protocol_info.config.max_notification_size,
);
self.events_queue.push_back(ProtocolsHandlerEvent::OutboundSubstreamRequest {
protocol: SubstreamProtocol::new(proto, protocol_index)
.with_timeout(OPEN_TIMEOUT),
});
self.events_queue.push_back(
ProtocolsHandlerEvent::OutboundSubstreamRequest {
protocol: SubstreamProtocol::new(proto, protocol_index)
.with_timeout(OPEN_TIMEOUT),
},
);
}
in_substream.send_handshake(handshake_message);
// The state change is done in two steps because of borrowing issues.
let in_substream = match
mem::replace(&mut protocol_info.state, State::Opening { in_substream: None })
{
let in_substream = match mem::replace(
&mut protocol_info.state,
State::Opening { in_substream: None },
) {
State::OpenDesiredByRemote { in_substream, .. } => in_substream,
_ => unreachable!()
};
protocol_info.state = State::Opening {
in_substream: Some(in_substream),
_ => unreachable!(),
};
protocol_info.state = State::Opening { in_substream: Some(in_substream) };
},
State::Opening { .. } |
State::Open { .. } => {
State::Opening { .. } | State::Open { .. } => {
// As documented, it is forbidden to send an `Open` while there is already
// one in the fly.
error!(target: "sub-libp2p", "opening already-opened handler");
@@ -642,34 +646,26 @@ impl ProtocolsHandler for NotifsHandler {
NotifsHandlerIn::Close { protocol_index } => {
match self.protocols[protocol_index].state {
State::Open { .. } => {
self.protocols[protocol_index].state = State::Closed {
pending_opening: false,
};
self.protocols[protocol_index].state =
State::Closed { pending_opening: false };
},
State::Opening { .. } => {
self.protocols[protocol_index].state = State::Closed {
pending_opening: true,
};
self.protocols[protocol_index].state =
State::Closed { pending_opening: true };
self.events_queue.push_back(ProtocolsHandlerEvent::Custom(
NotifsHandlerOut::OpenResultErr {
protocol_index,
}
NotifsHandlerOut::OpenResultErr { protocol_index },
));
},
State::OpenDesiredByRemote { pending_opening, .. } => {
self.protocols[protocol_index].state = State::Closed {
pending_opening,
};
}
self.protocols[protocol_index].state = State::Closed { pending_opening };
},
State::Closed { .. } => {},
}
self.events_queue.push_back(
ProtocolsHandlerEvent::Custom(NotifsHandlerOut::CloseResult {
protocol_index,
})
);
self.events_queue.push_back(ProtocolsHandlerEvent::Custom(
NotifsHandlerOut::CloseResult { protocol_index },
));
},
}
}
@@ -677,26 +673,22 @@ impl ProtocolsHandler for NotifsHandler {
fn inject_dial_upgrade_error(
&mut self,
num: usize,
_: ProtocolsHandlerUpgrErr<NotificationsHandshakeError>
_: ProtocolsHandlerUpgrErr<NotificationsHandshakeError>,
) {
match self.protocols[num].state {
State::Closed { ref mut pending_opening } |
State::OpenDesiredByRemote { ref mut pending_opening, .. } => {
debug_assert!(*pending_opening);
*pending_opening = false;
}
},
State::Opening { .. } => {
self.protocols[num].state = State::Closed {
pending_opening: false,
};
self.protocols[num].state = State::Closed { pending_opening: false };
self.events_queue.push_back(ProtocolsHandlerEvent::Custom(
NotifsHandlerOut::OpenResultErr {
protocol_index: num,
}
NotifsHandlerOut::OpenResultErr { protocol_index: num },
));
}
},
// No substream is being open when already `Open`.
State::Open { .. } => debug_assert!(false),
@@ -706,7 +698,7 @@ impl ProtocolsHandler for NotifsHandler {
fn connection_keep_alive(&self) -> KeepAlive {
// `Yes` if any protocol has some activity.
if self.protocols.iter().any(|p| !matches!(p.state, State::Closed { .. })) {
return KeepAlive::Yes;
return KeepAlive::Yes
}
// A grace period of `INITIAL_KEEPALIVE_TIME` must be given to leave time for the remote
@@ -718,28 +710,33 @@ impl ProtocolsHandler for NotifsHandler {
&mut self,
cx: &mut Context,
) -> Poll<
ProtocolsHandlerEvent<Self::OutboundProtocol, Self::OutboundOpenInfo, Self::OutEvent, Self::Error>
ProtocolsHandlerEvent<
Self::OutboundProtocol,
Self::OutboundOpenInfo,
Self::OutEvent,
Self::Error,
>,
> {
if let Some(ev) = self.events_queue.pop_front() {
return Poll::Ready(ev);
return Poll::Ready(ev)
}
// For each open substream, try send messages from `notifications_sink_rx` to the
// substream.
for protocol_index in 0..self.protocols.len() {
if let State::Open { notifications_sink_rx, out_substream: Some(out_substream), .. }
= &mut self.protocols[protocol_index].state
if let State::Open {
notifications_sink_rx, out_substream: Some(out_substream), ..
} = &mut self.protocols[protocol_index].state
{
loop {
// Only proceed with `out_substream.poll_ready_unpin` if there is an element
// available in `notifications_sink_rx`. This avoids waking up the task when
// a substream is ready to send if there isn't actually something to send.
match Pin::new(&mut *notifications_sink_rx).as_mut().poll_peek(cx) {
Poll::Ready(Some(&NotificationsSinkMessage::ForceClose)) => {
return Poll::Ready(
ProtocolsHandlerEvent::Close(NotifsHandlerError::SyncNotificationsClogged)
);
},
Poll::Ready(Some(&NotificationsSinkMessage::ForceClose)) =>
return Poll::Ready(ProtocolsHandlerEvent::Close(
NotifsHandlerError::SyncNotificationsClogged,
)),
Poll::Ready(Some(&NotificationsSinkMessage::Notification { .. })) => {},
Poll::Ready(None) | Poll::Pending => break,
}
@@ -748,19 +745,20 @@ impl ProtocolsHandler for NotifsHandler {
// substream is ready to accept a message.
match out_substream.poll_ready_unpin(cx) {
Poll::Ready(_) => {},
Poll::Pending => break
Poll::Pending => break,
}
// Now that the substream is ready for a message, grab what to send.
let message = match notifications_sink_rx.poll_next_unpin(cx) {
Poll::Ready(Some(NotificationsSinkMessage::Notification { message })) => message,
Poll::Ready(Some(NotificationsSinkMessage::ForceClose))
| Poll::Ready(None)
| Poll::Pending => {
Poll::Ready(Some(NotificationsSinkMessage::Notification { message })) =>
message,
Poll::Ready(Some(NotificationsSinkMessage::ForceClose)) |
Poll::Ready(None) |
Poll::Pending => {
// Should never be reached, as per `poll_peek` above.
debug_assert!(false);
break;
}
break
},
};
let _ = out_substream.start_send_unpin(message);
@@ -784,15 +782,15 @@ impl ProtocolsHandler for NotifsHandler {
Poll::Ready(Err(_)) => {
*out_substream = None;
let event = NotifsHandlerOut::CloseDesired { protocol_index };
return Poll::Ready(ProtocolsHandlerEvent::Custom(event));
}
return Poll::Ready(ProtocolsHandlerEvent::Custom(event))
},
};
}
},
State::Closed { .. } |
State::Opening { .. } |
State::Open { out_substream: None, .. } |
State::OpenDesiredByRemote { .. } => {}
State::OpenDesiredByRemote { .. } => {},
}
}
@@ -803,45 +801,40 @@ impl ProtocolsHandler for NotifsHandler {
match &mut self.protocols[protocol_index].state {
State::Closed { .. } |
State::Open { in_substream: None, .. } |
State::Opening { in_substream: None } => {}
State::Opening { in_substream: None } => {},
State::Open { in_substream: in_substream @ Some(_), .. } => {
State::Open { in_substream: in_substream @ Some(_), .. } =>
match Stream::poll_next(Pin::new(in_substream.as_mut().unwrap()), cx) {
Poll::Pending => {},
Poll::Ready(Some(Ok(message))) => {
let event = NotifsHandlerOut::Notification {
protocol_index,
message,
};
let event = NotifsHandlerOut::Notification { protocol_index, message };
return Poll::Ready(ProtocolsHandlerEvent::Custom(event))
},
Poll::Ready(None) | Poll::Ready(Some(Err(_))) =>
*in_substream = None,
}
}
Poll::Ready(None) | Poll::Ready(Some(Err(_))) => *in_substream = None,
},
State::OpenDesiredByRemote { in_substream, pending_opening } => {
State::OpenDesiredByRemote { in_substream, pending_opening } =>
match NotificationsInSubstream::poll_process(Pin::new(in_substream), cx) {
Poll::Pending => {},
Poll::Ready(Ok(void)) => match void {},
Poll::Ready(Err(_)) => {
self.protocols[protocol_index].state = State::Closed {
pending_opening: *pending_opening,
};
self.protocols[protocol_index].state =
State::Closed { pending_opening: *pending_opening };
return Poll::Ready(ProtocolsHandlerEvent::Custom(
NotifsHandlerOut::CloseDesired { protocol_index }
NotifsHandlerOut::CloseDesired { protocol_index },
))
},
}
}
},
State::Opening { in_substream: in_substream @ Some(_), .. } => {
match NotificationsInSubstream::poll_process(Pin::new(in_substream.as_mut().unwrap()), cx) {
State::Opening { in_substream: in_substream @ Some(_), .. } =>
match NotificationsInSubstream::poll_process(
Pin::new(in_substream.as_mut().unwrap()),
cx,
) {
Poll::Pending => {},
Poll::Ready(Ok(void)) => match void {},
Poll::Ready(Err(_)) => *in_substream = None,
}
}
},
}
}
@@ -21,19 +21,24 @@
use crate::protocol::notifications::{Notifications, NotificationsOut, ProtocolConfig};
use futures::prelude::*;
use libp2p::{PeerId, Multiaddr, Transport};
use libp2p::core::{
connection::{ConnectionId, ListenerId},
ConnectedPoint,
transport::MemoryTransport,
upgrade
use libp2p::{
core::{
connection::{ConnectionId, ListenerId},
transport::MemoryTransport,
upgrade, ConnectedPoint,
},
identity, noise,
swarm::{
IntoProtocolsHandler, NetworkBehaviour, NetworkBehaviourAction, PollParameters,
ProtocolsHandler, Swarm,
},
yamux, Multiaddr, PeerId, Transport,
};
use libp2p::{identity, noise, yamux};
use libp2p::swarm::{
Swarm, ProtocolsHandler, IntoProtocolsHandler, PollParameters,
NetworkBehaviour, NetworkBehaviourAction
use std::{
error, io, iter,
task::{Context, Poll},
time::Duration,
};
use std::{error, io, iter, task::{Context, Poll}, time::Duration};
/// Builds two nodes that have each other as bootstrap nodes.
/// This is to be used only for testing, and a panic will happen if something goes wrong.
@@ -45,12 +50,11 @@ fn build_nodes() -> (Swarm<CustomProtoWithAddr>, Swarm<CustomProtoWithAddr>) {
.map(|_| format!("/memory/{}", rand::random::<u64>()).parse().unwrap())
.collect();
for index in 0 .. 2 {
for index in 0..2 {
let keypair = keypairs[index].clone();
let noise_keys = noise::Keypair::<noise::X25519Spec>::new()
.into_authentic(&keypair)
.unwrap();
let noise_keys =
noise::Keypair::<noise::X25519Spec>::new().into_authentic(&keypair).unwrap();
let transport = MemoryTransport
.upgrade(upgrade::Version::V1)
@@ -60,48 +64,43 @@ fn build_nodes() -> (Swarm<CustomProtoWithAddr>, Swarm<CustomProtoWithAddr>) {
.boxed();
let (peerset, _) = sc_peerset::Peerset::from_config(sc_peerset::PeersetConfig {
sets: vec![
sc_peerset::SetConfig {
in_peers: 25,
out_peers: 25,
bootnodes: if index == 0 {
keypairs
.iter()
.skip(1)
.map(|keypair| keypair.public().into_peer_id())
.collect()
} else {
vec![]
},
reserved_nodes: Default::default(),
reserved_only: false,
}
],
sets: vec![sc_peerset::SetConfig {
in_peers: 25,
out_peers: 25,
bootnodes: if index == 0 {
keypairs.iter().skip(1).map(|keypair| keypair.public().into_peer_id()).collect()
} else {
vec![]
},
reserved_nodes: Default::default(),
reserved_only: false,
}],
});
let behaviour = CustomProtoWithAddr {
inner: Notifications::new(peerset, iter::once(ProtocolConfig {
name: "/foo".into(),
fallback_names: Vec::new(),
handshake: Vec::new(),
max_notification_size: 1024 * 1024
})),
inner: Notifications::new(
peerset,
iter::once(ProtocolConfig {
name: "/foo".into(),
fallback_names: Vec::new(),
handshake: Vec::new(),
max_notification_size: 1024 * 1024,
}),
),
addrs: addrs
.iter()
.enumerate()
.filter_map(|(n, a)| if n != index {
Some((keypairs[n].public().into_peer_id(), a.clone()))
} else {
None
.filter_map(|(n, a)| {
if n != index {
Some((keypairs[n].public().into_peer_id(), a.clone()))
} else {
None
}
})
.collect(),
};
let mut swarm = Swarm::new(
transport,
behaviour,
keypairs[index].public().into_peer_id()
);
let mut swarm = Swarm::new(transport, behaviour, keypairs[index].public().into_peer_id());
swarm.listen_on(addrs[index].clone()).unwrap();
out.push(swarm);
}
@@ -159,11 +158,21 @@ impl NetworkBehaviour for CustomProtoWithAddr {
self.inner.inject_disconnected(peer_id)
}
fn inject_connection_established(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) {
fn inject_connection_established(
&mut self,
peer_id: &PeerId,
conn: &ConnectionId,
endpoint: &ConnectedPoint,
) {
self.inner.inject_connection_established(peer_id, conn, endpoint)
}
fn inject_connection_closed(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) {
fn inject_connection_closed(
&mut self,
peer_id: &PeerId,
conn: &ConnectionId,
endpoint: &ConnectedPoint,
) {
self.inner.inject_connection_closed(peer_id, conn, endpoint)
}
@@ -171,7 +180,7 @@ impl NetworkBehaviour for CustomProtoWithAddr {
&mut self,
peer_id: PeerId,
connection: ConnectionId,
event: <<Self::ProtocolsHandler as IntoProtocolsHandler>::Handler as ProtocolsHandler>::OutEvent
event: <<Self::ProtocolsHandler as IntoProtocolsHandler>::Handler as ProtocolsHandler>::OutEvent,
) {
self.inner.inject_event(peer_id, connection, event)
}
@@ -185,11 +194,16 @@ impl NetworkBehaviour for CustomProtoWithAddr {
<<Self::ProtocolsHandler as IntoProtocolsHandler>::Handler as ProtocolsHandler>::InEvent,
Self::OutEvent
>
> {
>{
self.inner.poll(cx, params)
}
fn inject_addr_reach_failure(&mut self, peer_id: Option<&PeerId>, addr: &Multiaddr, error: &dyn std::error::Error) {
fn inject_addr_reach_failure(
&mut self,
peer_id: Option<&PeerId>,
addr: &Multiaddr,
error: &dyn std::error::Error,
) {
self.inner.inject_addr_reach_failure(peer_id, addr, error)
}
@@ -235,7 +249,12 @@ fn reconnect_after_disconnect() {
// For this test, the services can be in the following states.
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
enum ServiceState { NotConnected, FirstConnec, Disconnected, ConnectedAgain }
enum ServiceState {
NotConnected,
FirstConnec,
Disconnected,
ConnectedAgain,
}
let mut service1_state = ServiceState::NotConnected;
let mut service2_state = ServiceState::NotConnected;
@@ -253,55 +272,55 @@ fn reconnect_after_disconnect() {
};
match event {
future::Either::Left(NotificationsOut::CustomProtocolOpen { .. }) => {
future::Either::Left(NotificationsOut::CustomProtocolOpen { .. }) =>
match service1_state {
ServiceState::NotConnected => {
service1_state = ServiceState::FirstConnec;
if service2_state == ServiceState::FirstConnec {
service1.behaviour_mut().disconnect_peer(
Swarm::local_peer_id(&service2),
sc_peerset::SetId::from(0)
sc_peerset::SetId::from(0),
);
}
},
ServiceState::Disconnected => service1_state = ServiceState::ConnectedAgain,
ServiceState::FirstConnec | ServiceState::ConnectedAgain => panic!(),
}
},
future::Either::Left(NotificationsOut::CustomProtocolClosed { .. }) => {
},
future::Either::Left(NotificationsOut::CustomProtocolClosed { .. }) =>
match service1_state {
ServiceState::FirstConnec => service1_state = ServiceState::Disconnected,
ServiceState::ConnectedAgain| ServiceState::NotConnected |
ServiceState::ConnectedAgain |
ServiceState::NotConnected |
ServiceState::Disconnected => panic!(),
}
},
future::Either::Right(NotificationsOut::CustomProtocolOpen { .. }) => {
},
future::Either::Right(NotificationsOut::CustomProtocolOpen { .. }) =>
match service2_state {
ServiceState::NotConnected => {
service2_state = ServiceState::FirstConnec;
if service1_state == ServiceState::FirstConnec {
service1.behaviour_mut().disconnect_peer(
Swarm::local_peer_id(&service2),
sc_peerset::SetId::from(0)
sc_peerset::SetId::from(0),
);
}
},
ServiceState::Disconnected => service2_state = ServiceState::ConnectedAgain,
ServiceState::FirstConnec | ServiceState::ConnectedAgain => panic!(),
}
},
future::Either::Right(NotificationsOut::CustomProtocolClosed { .. }) => {
},
future::Either::Right(NotificationsOut::CustomProtocolClosed { .. }) =>
match service2_state {
ServiceState::FirstConnec => service2_state = ServiceState::Disconnected,
ServiceState::ConnectedAgain| ServiceState::NotConnected |
ServiceState::ConnectedAgain |
ServiceState::NotConnected |
ServiceState::Disconnected => panic!(),
}
},
_ => {}
},
_ => {},
}
if service1_state == ServiceState::ConnectedAgain && service2_state == ServiceState::ConnectedAgain {
break;
if service1_state == ServiceState::ConnectedAgain &&
service2_state == ServiceState::ConnectedAgain
{
break
}
}
@@ -316,7 +335,7 @@ fn reconnect_after_disconnect() {
let s2 = service2.next();
futures::pin_mut!(s1, s2);
match future::select(future::select(s1, s2), &mut delay).await {
future::Either::Right(_) => break, // success
future::Either::Right(_) => break, // success
future::Either::Left((future::Either::Left((ev, _)), _)) => ev,
future::Either::Left((future::Either::Right((ev, _)), _)) => ev,
}
@@ -325,7 +344,7 @@ fn reconnect_after_disconnect() {
match event {
NotificationsOut::CustomProtocolOpen { .. } |
NotificationsOut::CustomProtocolClosed { .. } => panic!(),
_ => {}
_ => {},
}
}
});
@@ -16,16 +16,13 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
pub use self::collec::UpgradeCollec;
pub use self::notifications::{
NotificationsIn,
NotificationsInOpen,
NotificationsInSubstream,
NotificationsOut,
NotificationsOutOpen,
NotificationsOutSubstream,
NotificationsHandshakeError,
NotificationsOutError,
pub use self::{
collec::UpgradeCollec,
notifications::{
NotificationsHandshakeError, NotificationsIn, NotificationsInOpen,
NotificationsInSubstream, NotificationsOut, NotificationsOutError, NotificationsOutOpen,
NotificationsOutSubstream,
},
};
mod collec;
@@ -18,7 +18,12 @@
use futures::prelude::*;
use libp2p::core::upgrade::{InboundUpgrade, ProtocolName, UpgradeInfo};
use std::{iter::FromIterator, pin::Pin, task::{Context, Poll}, vec};
use std::{
iter::FromIterator,
pin::Pin,
task::{Context, Poll},
vec,
};
// TODO: move this to libp2p => https://github.com/libp2p/rust-libp2p/issues/1445
@@ -44,9 +49,10 @@ impl<T: UpgradeInfo> UpgradeInfo for UpgradeCollec<T> {
type InfoIter = vec::IntoIter<Self::Info>;
fn protocol_info(&self) -> Self::InfoIter {
self.0.iter().enumerate()
.flat_map(|(n, p)|
p.protocol_info().into_iter().map(move |i| ProtoNameWithUsize(i, n)))
self.0
.iter()
.enumerate()
.flat_map(|(n, p)| p.protocol_info().into_iter().map(move |i| ProtoNameWithUsize(i, n)))
.collect::<Vec<_>>()
.into_iter()
}
@@ -16,6 +16,7 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use asynchronous_codec::Framed;
/// Notifications protocol.
///
/// The Substrate notifications protocol consists in the following:
@@ -34,14 +35,18 @@
///
/// Notification substreams are unidirectional. If A opens a substream with B, then B is
/// encouraged but not required to open a substream to A as well.
///
use bytes::BytesMut;
use futures::prelude::*;
use asynchronous_codec::Framed;
use libp2p::core::{UpgradeInfo, InboundUpgrade, OutboundUpgrade, upgrade};
use libp2p::core::{upgrade, InboundUpgrade, OutboundUpgrade, UpgradeInfo};
use log::error;
use std::{borrow::Cow, convert::{Infallible, TryFrom as _}, io, mem, pin::Pin, task::{Context, Poll}, vec};
use std::{
borrow::Cow,
convert::{Infallible, TryFrom as _},
io, mem,
pin::Pin,
task::{Context, Poll},
vec,
};
use unsigned_varint::codec::UviBytes;
/// Maximum allowed size of the two handshake messages, in bytes.
@@ -111,15 +116,12 @@ impl NotificationsIn {
pub fn new(
main_protocol_name: impl Into<Cow<'static, str>>,
fallback_names: Vec<Cow<'static, str>>,
max_notification_size: u64
max_notification_size: u64,
) -> Self {
let mut protocol_names = fallback_names;
protocol_names.insert(0, main_protocol_name.into());
NotificationsIn {
protocol_names,
max_notification_size,
}
NotificationsIn { protocol_names, max_notification_size }
}
}
@@ -128,29 +130,31 @@ impl UpgradeInfo for NotificationsIn {
type InfoIter = vec::IntoIter<Self::Info>;
fn protocol_info(&self) -> Self::InfoIter {
self.protocol_names.iter().cloned().map(StringProtocolName).collect::<Vec<_>>().into_iter()
self.protocol_names
.iter()
.cloned()
.map(StringProtocolName)
.collect::<Vec<_>>()
.into_iter()
}
}
impl<TSubstream> InboundUpgrade<TSubstream> for NotificationsIn
where TSubstream: AsyncRead + AsyncWrite + Unpin + Send + 'static,
where
TSubstream: AsyncRead + AsyncWrite + Unpin + Send + 'static,
{
type Output = NotificationsInOpen<TSubstream>;
type Future = Pin<Box<dyn Future<Output = Result<Self::Output, Self::Error>> + Send>>;
type Error = NotificationsHandshakeError;
fn upgrade_inbound(
self,
mut socket: TSubstream,
negotiated_name: Self::Info,
) -> Self::Future {
fn upgrade_inbound(self, mut socket: TSubstream, negotiated_name: Self::Info) -> Self::Future {
Box::pin(async move {
let handshake_len = unsigned_varint::aio::read_usize(&mut socket).await?;
if handshake_len > MAX_HANDSHAKE_SIZE {
return Err(NotificationsHandshakeError::TooLarge {
requested: handshake_len,
max: MAX_HANDSHAKE_SIZE,
});
})
}
let mut handshake = vec![0u8; handshake_len];
@@ -191,13 +195,14 @@ pub struct NotificationsInOpen<TSubstream> {
}
impl<TSubstream> NotificationsInSubstream<TSubstream>
where TSubstream: AsyncRead + AsyncWrite + Unpin,
where
TSubstream: AsyncRead + AsyncWrite + Unpin,
{
/// Sends the handshake in order to inform the remote that we accept the substream.
pub fn send_handshake(&mut self, message: impl Into<Vec<u8>>) {
if !matches!(self.handshake, NotificationsInSubstreamHandshake::NotSent) {
error!(target: "sub-libp2p", "Tried to send handshake twice");
return;
return
}
self.handshake = NotificationsInSubstreamHandshake::PendingSend(message.into());
@@ -205,7 +210,10 @@ where TSubstream: AsyncRead + AsyncWrite + Unpin,
/// Equivalent to `Stream::poll_next`, except that it only drives the handshake and is
/// guaranteed to not generate any notification.
pub fn poll_process(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Result<Infallible, io::Error>> {
pub fn poll_process(
self: Pin<&mut Self>,
cx: &mut Context,
) -> Poll<Result<Infallible, io::Error>> {
let mut this = self.project();
loop {
@@ -222,7 +230,7 @@ where TSubstream: AsyncRead + AsyncWrite + Unpin,
Poll::Pending => {
*this.handshake = NotificationsInSubstreamHandshake::PendingSend(msg);
return Poll::Pending
}
},
},
NotificationsInSubstreamHandshake::Flush =>
match Sink::poll_flush(this.socket.as_mut(), cx)? {
@@ -231,7 +239,7 @@ where TSubstream: AsyncRead + AsyncWrite + Unpin,
Poll::Pending => {
*this.handshake = NotificationsInSubstreamHandshake::Flush;
return Poll::Pending
}
},
},
st @ NotificationsInSubstreamHandshake::NotSent |
@@ -239,15 +247,16 @@ where TSubstream: AsyncRead + AsyncWrite + Unpin,
st @ NotificationsInSubstreamHandshake::ClosingInResponseToRemote |
st @ NotificationsInSubstreamHandshake::BothSidesClosed => {
*this.handshake = st;
return Poll::Pending;
}
return Poll::Pending
},
}
}
}
}
impl<TSubstream> Stream for NotificationsInSubstream<TSubstream>
where TSubstream: AsyncRead + AsyncWrite + Unpin,
where
TSubstream: AsyncRead + AsyncWrite + Unpin,
{
type Item = Result<BytesMut, io::Error>;
@@ -273,7 +282,7 @@ where TSubstream: AsyncRead + AsyncWrite + Unpin,
Poll::Pending => {
*this.handshake = NotificationsInSubstreamHandshake::PendingSend(msg);
return Poll::Pending
}
},
},
NotificationsInSubstreamHandshake::Flush =>
match Sink::poll_flush(this.socket.as_mut(), cx)? {
@@ -282,13 +291,14 @@ where TSubstream: AsyncRead + AsyncWrite + Unpin,
Poll::Pending => {
*this.handshake = NotificationsInSubstreamHandshake::Flush;
return Poll::Pending
}
},
},
NotificationsInSubstreamHandshake::Sent => {
match Stream::poll_next(this.socket.as_mut(), cx) {
Poll::Ready(None) => *this.handshake =
NotificationsInSubstreamHandshake::ClosingInResponseToRemote,
Poll::Ready(None) =>
*this.handshake =
NotificationsInSubstreamHandshake::ClosingInResponseToRemote,
Poll::Ready(Some(msg)) => {
*this.handshake = NotificationsInSubstreamHandshake::Sent;
return Poll::Ready(Some(msg))
@@ -305,13 +315,13 @@ where TSubstream: AsyncRead + AsyncWrite + Unpin,
Poll::Ready(()) =>
*this.handshake = NotificationsInSubstreamHandshake::BothSidesClosed,
Poll::Pending => {
*this.handshake = NotificationsInSubstreamHandshake::ClosingInResponseToRemote;
*this.handshake =
NotificationsInSubstreamHandshake::ClosingInResponseToRemote;
return Poll::Pending
}
},
},
NotificationsInSubstreamHandshake::BothSidesClosed =>
return Poll::Ready(None),
NotificationsInSubstreamHandshake::BothSidesClosed => return Poll::Ready(None),
}
}
}
@@ -333,11 +343,7 @@ impl NotificationsOut {
let mut protocol_names = fallback_names;
protocol_names.insert(0, main_protocol_name.into());
NotificationsOut {
protocol_names,
initial_message,
max_notification_size,
}
NotificationsOut { protocol_names, initial_message, max_notification_size }
}
}
@@ -356,22 +362,24 @@ impl UpgradeInfo for NotificationsOut {
type InfoIter = vec::IntoIter<Self::Info>;
fn protocol_info(&self) -> Self::InfoIter {
self.protocol_names.iter().cloned().map(StringProtocolName).collect::<Vec<_>>().into_iter()
self.protocol_names
.iter()
.cloned()
.map(StringProtocolName)
.collect::<Vec<_>>()
.into_iter()
}
}
impl<TSubstream> OutboundUpgrade<TSubstream> for NotificationsOut
where TSubstream: AsyncRead + AsyncWrite + Unpin + Send + 'static,
where
TSubstream: AsyncRead + AsyncWrite + Unpin + Send + 'static,
{
type Output = NotificationsOutOpen<TSubstream>;
type Future = Pin<Box<dyn Future<Output = Result<Self::Output, Self::Error>> + Send>>;
type Error = NotificationsHandshakeError;
fn upgrade_outbound(
self,
mut socket: TSubstream,
negotiated_name: Self::Info,
) -> Self::Future {
fn upgrade_outbound(self, mut socket: TSubstream, negotiated_name: Self::Info) -> Self::Future {
Box::pin(async move {
upgrade::write_with_len_prefix(&mut socket, &self.initial_message).await?;
@@ -381,7 +389,7 @@ where TSubstream: AsyncRead + AsyncWrite + Unpin + Send + 'static,
return Err(NotificationsHandshakeError::TooLarge {
requested: handshake_len,
max: MAX_HANDSHAKE_SIZE,
});
})
}
let mut handshake = vec![0u8; handshake_len];
@@ -399,9 +407,7 @@ where TSubstream: AsyncRead + AsyncWrite + Unpin + Send + 'static,
} else {
Some(negotiated_name.0)
},
substream: NotificationsOutSubstream {
socket: Framed::new(socket, codec),
}
substream: NotificationsOutSubstream { socket: Framed::new(socket, codec) },
})
})
}
@@ -419,14 +425,14 @@ pub struct NotificationsOutOpen<TSubstream> {
}
impl<TSubstream> Sink<Vec<u8>> for NotificationsOutSubstream<TSubstream>
where TSubstream: AsyncRead + AsyncWrite + Unpin,
where
TSubstream: AsyncRead + AsyncWrite + Unpin,
{
type Error = NotificationsOutError;
fn poll_ready(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Result<(), Self::Error>> {
let mut this = self.project();
Sink::poll_ready(this.socket.as_mut(), cx)
.map_err(NotificationsOutError::Io)
Sink::poll_ready(this.socket.as_mut(), cx).map_err(NotificationsOutError::Io)
}
fn start_send(self: Pin<&mut Self>, item: Vec<u8>) -> Result<(), Self::Error> {
@@ -437,14 +443,12 @@ impl<TSubstream> Sink<Vec<u8>> for NotificationsOutSubstream<TSubstream>
fn poll_flush(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Result<(), Self::Error>> {
let mut this = self.project();
Sink::poll_flush(this.socket.as_mut(), cx)
.map_err(NotificationsOutError::Io)
Sink::poll_flush(this.socket.as_mut(), cx).map_err(NotificationsOutError::Io)
}
fn poll_close(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Result<(), Self::Error>> {
let mut this = self.project();
Sink::poll_close(this.socket.as_mut(), cx)
.map_err(NotificationsOutError::Io)
Sink::poll_close(this.socket.as_mut(), cx).map_err(NotificationsOutError::Io)
}
}
@@ -471,11 +475,12 @@ impl From<unsigned_varint::io::ReadError> for NotificationsHandshakeError {
fn from(err: unsigned_varint::io::ReadError) -> Self {
match err {
unsigned_varint::io::ReadError::Io(err) => NotificationsHandshakeError::Io(err),
unsigned_varint::io::ReadError::Decode(err) => NotificationsHandshakeError::VarintDecode(err),
unsigned_varint::io::ReadError::Decode(err) =>
NotificationsHandshakeError::VarintDecode(err),
_ => {
log::warn!("Unrecognized varint decoding error");
NotificationsHandshakeError::Io(From::from(io::ErrorKind::InvalidData))
}
},
}
}
}
@@ -492,7 +497,7 @@ mod tests {
use super::{NotificationsIn, NotificationsInOpen, NotificationsOut, NotificationsOutOpen};
use async_std::net::{TcpListener, TcpStream};
use futures::{prelude::*, channel::oneshot};
use futures::{channel::oneshot, prelude::*};
use libp2p::core::upgrade;
use std::borrow::Cow;
@@ -506,8 +511,10 @@ mod tests {
let NotificationsOutOpen { handshake, mut substream, .. } = upgrade::apply_outbound(
socket,
NotificationsOut::new(PROTO_NAME, Vec::new(), &b"initial message"[..], 1024 * 1024),
upgrade::Version::V1
).await.unwrap();
upgrade::Version::V1,
)
.await
.unwrap();
assert_eq!(handshake, b"hello world");
substream.send(b"test message".to_vec()).await.unwrap();
@@ -520,8 +527,10 @@ mod tests {
let (socket, _) = listener.accept().await.unwrap();
let NotificationsInOpen { handshake, mut substream, .. } = upgrade::apply_inbound(
socket,
NotificationsIn::new(PROTO_NAME, Vec::new(), 1024 * 1024)
).await.unwrap();
NotificationsIn::new(PROTO_NAME, Vec::new(), 1024 * 1024),
)
.await
.unwrap();
assert_eq!(handshake, b"initial message");
substream.send_handshake(&b"hello world"[..]);
@@ -545,8 +554,10 @@ mod tests {
let NotificationsOutOpen { handshake, mut substream, .. } = upgrade::apply_outbound(
socket,
NotificationsOut::new(PROTO_NAME, Vec::new(), vec![], 1024 * 1024),
upgrade::Version::V1
).await.unwrap();
upgrade::Version::V1,
)
.await
.unwrap();
assert!(handshake.is_empty());
substream.send(Default::default()).await.unwrap();
@@ -559,8 +570,10 @@ mod tests {
let (socket, _) = listener.accept().await.unwrap();
let NotificationsInOpen { handshake, mut substream, .. } = upgrade::apply_inbound(
socket,
NotificationsIn::new(PROTO_NAME, Vec::new(), 1024 * 1024)
).await.unwrap();
NotificationsIn::new(PROTO_NAME, Vec::new(), 1024 * 1024),
)
.await
.unwrap();
assert!(handshake.is_empty());
substream.send_handshake(vec![]);
@@ -582,8 +595,9 @@ mod tests {
let outcome = upgrade::apply_outbound(
socket,
NotificationsOut::new(PROTO_NAME, Vec::new(), &b"hello"[..], 1024 * 1024),
upgrade::Version::V1
).await;
upgrade::Version::V1,
)
.await;
// Despite the protocol negotiation being successfully conducted on the listener
// side, we have to receive an error here because the listener didn't send the
@@ -598,8 +612,10 @@ mod tests {
let (socket, _) = listener.accept().await.unwrap();
let NotificationsInOpen { handshake, substream, .. } = upgrade::apply_inbound(
socket,
NotificationsIn::new(PROTO_NAME, Vec::new(), 1024 * 1024)
).await.unwrap();
NotificationsIn::new(PROTO_NAME, Vec::new(), 1024 * 1024),
)
.await
.unwrap();
assert_eq!(handshake, b"hello");
@@ -620,9 +636,15 @@ mod tests {
let ret = upgrade::apply_outbound(
socket,
// We check that an initial message that is too large gets refused.
NotificationsOut::new(PROTO_NAME, Vec::new(), (0..32768).map(|_| 0).collect::<Vec<_>>(), 1024 * 1024),
upgrade::Version::V1
).await;
NotificationsOut::new(
PROTO_NAME,
Vec::new(),
(0..32768).map(|_| 0).collect::<Vec<_>>(),
1024 * 1024,
),
upgrade::Version::V1,
)
.await;
assert!(ret.is_err());
});
@@ -633,8 +655,9 @@ mod tests {
let (socket, _) = listener.accept().await.unwrap();
let ret = upgrade::apply_inbound(
socket,
NotificationsIn::new(PROTO_NAME, Vec::new(), 1024 * 1024)
).await;
NotificationsIn::new(PROTO_NAME, Vec::new(), 1024 * 1024),
)
.await;
assert!(ret.is_err());
});
@@ -651,8 +674,9 @@ mod tests {
let ret = upgrade::apply_outbound(
socket,
NotificationsOut::new(PROTO_NAME, Vec::new(), &b"initial message"[..], 1024 * 1024),
upgrade::Version::V1
).await;
upgrade::Version::V1,
)
.await;
assert!(ret.is_err());
});
@@ -663,8 +687,10 @@ mod tests {
let (socket, _) = listener.accept().await.unwrap();
let NotificationsInOpen { handshake, mut substream, .. } = upgrade::apply_inbound(
socket,
NotificationsIn::new(PROTO_NAME, Vec::new(), 1024 * 1024)
).await.unwrap();
NotificationsIn::new(PROTO_NAME, Vec::new(), 1024 * 1024),
)
.await
.unwrap();
assert_eq!(handshake, b"initial message");
// We check that a handshake that is too large gets refused.
File diff suppressed because it is too large Load Diff
@@ -16,13 +16,15 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use std::cmp;
use std::ops::Range;
use std::collections::{HashMap, BTreeMap};
use log::trace;
use libp2p::PeerId;
use sp_runtime::traits::{Block as BlockT, NumberFor, One};
use crate::protocol::message;
use libp2p::PeerId;
use log::trace;
use sp_runtime::traits::{Block as BlockT, NumberFor, One};
use std::{
cmp,
collections::{BTreeMap, HashMap},
ops::Range,
};
/// Block data with origin.
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -35,10 +37,7 @@ pub struct BlockData<B: BlockT> {
#[derive(Debug)]
enum BlockRangeState<B: BlockT> {
Downloading {
len: NumberFor<B>,
downloading: u32,
},
Downloading { len: NumberFor<B>, downloading: u32 },
Complete(Vec<BlockData<B>>),
}
@@ -62,10 +61,7 @@ pub struct BlockCollection<B: BlockT> {
impl<B: BlockT> BlockCollection<B> {
/// Create a new instance.
pub fn new() -> Self {
BlockCollection {
blocks: BTreeMap::new(),
peer_requests: HashMap::new(),
}
BlockCollection { blocks: BTreeMap::new(), peer_requests: HashMap::new() }
}
/// Clear everything.
@@ -77,7 +73,7 @@ impl<B: BlockT> BlockCollection<B> {
/// Insert a set of blocks into collection.
pub fn insert(&mut self, start: NumberFor<B>, blocks: Vec<message::BlockData<B>>, who: PeerId) {
if blocks.is_empty() {
return;
return
}
match self.blocks.get(&start) {
@@ -86,13 +82,20 @@ impl<B: BlockT> BlockCollection<B> {
},
Some(&BlockRangeState::Complete(ref existing)) if existing.len() >= blocks.len() => {
trace!(target: "sync", "Ignored block data already downloaded: {}", start);
return;
return
},
_ => (),
}
self.blocks.insert(start, BlockRangeState::Complete(blocks.into_iter()
.map(|b| BlockData { origin: Some(who.clone()), block: b }).collect()));
self.blocks.insert(
start,
BlockRangeState::Complete(
blocks
.into_iter()
.map(|b| BlockData { origin: Some(who.clone()), block: b })
.collect(),
),
);
}
/// Returns a set of block hashes that require a header download. The returned set is marked as being downloaded.
@@ -107,7 +110,7 @@ impl<B: BlockT> BlockCollection<B> {
) -> Option<Range<NumberFor<B>>> {
if peer_best <= common {
// Bail out early
return None;
return None
}
// First block number that we need to download
let first_different = common + <NumberFor<B>>::one();
@@ -120,15 +123,13 @@ impl<B: BlockT> BlockCollection<B> {
break match (prev, next) {
(Some((start, &BlockRangeState::Downloading { ref len, downloading })), _)
if downloading < max_parallel =>
(*start .. *start + *len, downloading),
(*start..*start + *len, downloading),
(Some((start, r)), Some((next_start, _))) if *start + r.len() < *next_start =>
(*start + r.len() .. cmp::min(*next_start, *start + r.len() + count), 0), // gap
(Some((start, r)), None) =>
(*start + r.len() .. *start + r.len() + count, 0), // last range
(None, None) =>
(first_different .. first_different + count, 0), // empty
(*start + r.len()..cmp::min(*next_start, *start + r.len() + count), 0), // gap
(Some((start, r)), None) => (*start + r.len()..*start + r.len() + count, 0), /* last range */
(None, None) => (first_different..first_different + count, 0), /* empty */
(None, Some((start, _))) if *start > first_different =>
(first_different .. cmp::min(first_different + count, *start), 0), // gap at the start
(first_different..cmp::min(first_different + count, *start), 0), /* gap at the start */
_ => {
prev = next;
continue
@@ -139,23 +140,33 @@ impl<B: BlockT> BlockCollection<B> {
// crop to peers best
if range.start > peer_best {
trace!(target: "sync", "Out of range for peer {} ({} vs {})", who, range.start, peer_best);
return None;
return None
}
range.end = cmp::min(peer_best + One::one(), range.end);
if self.blocks.iter().next().map_or(false, |(n, _)| range.start > *n + max_ahead.into()) {
if self
.blocks
.iter()
.next()
.map_or(false, |(n, _)| range.start > *n + max_ahead.into())
{
trace!(target: "sync", "Too far ahead for peer {} ({})", who, range.start);
return None;
return None
}
self.peer_requests.insert(who, range.start);
self.blocks.insert(range.start, BlockRangeState::Downloading {
len: range.end - range.start,
downloading: downloading + 1
});
self.blocks.insert(
range.start,
BlockRangeState::Downloading {
len: range.end - range.start,
downloading: downloading + 1,
},
);
if range.end <= range.start {
panic!("Empty range {:?}, count={}, peer_best={}, common={}, blocks={:?}",
range, count, peer_best, common, self.blocks);
panic!(
"Empty range {:?}, count={}, peer_best={}, common={}, blocks={:?}",
range, count, peer_best, common, self.blocks
);
}
Some(range)
}
@@ -188,16 +199,14 @@ impl<B: BlockT> BlockCollection<B> {
pub fn clear_peer_download(&mut self, who: &PeerId) {
if let Some(start) = self.peer_requests.remove(who) {
let remove = match self.blocks.get_mut(&start) {
Some(&mut BlockRangeState::Downloading { ref mut downloading, .. }) if *downloading > 1 => {
Some(&mut BlockRangeState::Downloading { ref mut downloading, .. })
if *downloading > 1 =>
{
*downloading -= 1;
false
},
Some(&mut BlockRangeState::Downloading { .. }) => {
true
},
_ => {
false
}
Some(&mut BlockRangeState::Downloading { .. }) => true,
_ => false,
};
if remove {
self.blocks.remove(&start);
@@ -210,27 +219,28 @@ impl<B: BlockT> BlockCollection<B> {
mod test {
use super::{BlockCollection, BlockData, BlockRangeState};
use crate::{protocol::message, PeerId};
use sp_runtime::testing::{Block as RawBlock, ExtrinsicWrapper};
use sp_core::H256;
use sp_runtime::testing::{Block as RawBlock, ExtrinsicWrapper};
type Block = RawBlock<ExtrinsicWrapper<u64>>;
fn is_empty(bc: &BlockCollection<Block>) -> bool {
bc.blocks.is_empty() &&
bc.peer_requests.is_empty()
bc.blocks.is_empty() && bc.peer_requests.is_empty()
}
fn generate_blocks(n: usize) -> Vec<message::BlockData<Block>> {
(0 .. n).map(|_| message::generic::BlockData {
hash: H256::random(),
header: None,
body: None,
indexed_body: None,
message_queue: None,
receipt: None,
justification: None,
justifications: None,
}).collect()
(0..n)
.map(|_| message::generic::BlockData {
hash: H256::random(),
header: None,
body: None,
indexed_body: None,
message_queue: None,
receipt: None,
justification: None,
justifications: None,
})
.collect()
}
#[test]
@@ -252,32 +262,47 @@ mod test {
let peer2 = PeerId::random();
let blocks = generate_blocks(150);
assert_eq!(bc.needed_blocks(peer0.clone(), 40, 150, 0, 1, 200), Some(1 .. 41));
assert_eq!(bc.needed_blocks(peer1.clone(), 40, 150, 0, 1, 200), Some(41 .. 81));
assert_eq!(bc.needed_blocks(peer2.clone(), 40, 150, 0, 1, 200), Some(81 .. 121));
assert_eq!(bc.needed_blocks(peer0.clone(), 40, 150, 0, 1, 200), Some(1..41));
assert_eq!(bc.needed_blocks(peer1.clone(), 40, 150, 0, 1, 200), Some(41..81));
assert_eq!(bc.needed_blocks(peer2.clone(), 40, 150, 0, 1, 200), Some(81..121));
bc.clear_peer_download(&peer1);
bc.insert(41, blocks[41..81].to_vec(), peer1.clone());
assert_eq!(bc.drain(1), vec![]);
assert_eq!(bc.needed_blocks(peer1.clone(), 40, 150, 0, 1, 200), Some(121 .. 151));
assert_eq!(bc.needed_blocks(peer1.clone(), 40, 150, 0, 1, 200), Some(121..151));
bc.clear_peer_download(&peer0);
bc.insert(1, blocks[1..11].to_vec(), peer0.clone());
assert_eq!(bc.needed_blocks(peer0.clone(), 40, 150, 0, 1, 200), Some(11 .. 41));
assert_eq!(bc.drain(1), blocks[1..11].iter()
.map(|b| BlockData { block: b.clone(), origin: Some(peer0.clone()) }).collect::<Vec<_>>());
assert_eq!(bc.needed_blocks(peer0.clone(), 40, 150, 0, 1, 200), Some(11..41));
assert_eq!(
bc.drain(1),
blocks[1..11]
.iter()
.map(|b| BlockData { block: b.clone(), origin: Some(peer0.clone()) })
.collect::<Vec<_>>()
);
bc.clear_peer_download(&peer0);
bc.insert(11, blocks[11..41].to_vec(), peer0.clone());
let drained = bc.drain(12);
assert_eq!(drained[..30], blocks[11..41].iter()
.map(|b| BlockData { block: b.clone(), origin: Some(peer0.clone()) }).collect::<Vec<_>>()[..]);
assert_eq!(drained[30..], blocks[41..81].iter()
.map(|b| BlockData { block: b.clone(), origin: Some(peer1.clone()) }).collect::<Vec<_>>()[..]);
assert_eq!(
drained[..30],
blocks[11..41]
.iter()
.map(|b| BlockData { block: b.clone(), origin: Some(peer0.clone()) })
.collect::<Vec<_>>()[..]
);
assert_eq!(
drained[30..],
blocks[41..81]
.iter()
.map(|b| BlockData { block: b.clone(), origin: Some(peer1.clone()) })
.collect::<Vec<_>>()[..]
);
bc.clear_peer_download(&peer2);
assert_eq!(bc.needed_blocks(peer2.clone(), 40, 150, 80, 1, 200), Some(81 .. 121));
assert_eq!(bc.needed_blocks(peer2.clone(), 40, 150, 80, 1, 200), Some(81..121));
bc.clear_peer_download(&peer2);
bc.insert(81, blocks[81..121].to_vec(), peer2.clone());
bc.clear_peer_download(&peer1);
@@ -285,25 +310,38 @@ mod test {
assert_eq!(bc.drain(80), vec![]);
let drained = bc.drain(81);
assert_eq!(drained[..40], blocks[81..121].iter()
.map(|b| BlockData { block: b.clone(), origin: Some(peer2.clone()) }).collect::<Vec<_>>()[..]);
assert_eq!(drained[40..], blocks[121..150].iter()
.map(|b| BlockData { block: b.clone(), origin: Some(peer1.clone()) }).collect::<Vec<_>>()[..]);
assert_eq!(
drained[..40],
blocks[81..121]
.iter()
.map(|b| BlockData { block: b.clone(), origin: Some(peer2.clone()) })
.collect::<Vec<_>>()[..]
);
assert_eq!(
drained[40..],
blocks[121..150]
.iter()
.map(|b| BlockData { block: b.clone(), origin: Some(peer1.clone()) })
.collect::<Vec<_>>()[..]
);
}
#[test]
fn large_gap() {
let mut bc: BlockCollection<Block> = BlockCollection::new();
bc.blocks.insert(100, BlockRangeState::Downloading {
len: 128,
downloading: 1,
});
let blocks = generate_blocks(10).into_iter().map(|b| BlockData { block: b, origin: None }).collect();
bc.blocks.insert(100, BlockRangeState::Downloading { len: 128, downloading: 1 });
let blocks = generate_blocks(10)
.into_iter()
.map(|b| BlockData { block: b, origin: None })
.collect();
bc.blocks.insert(114305, BlockRangeState::Complete(blocks));
let peer0 = PeerId::random();
assert_eq!(bc.needed_blocks(peer0.clone(), 128, 10000, 000, 1, 200), Some(1 .. 100));
assert_eq!(bc.needed_blocks(peer0.clone(), 128, 10000, 000, 1, 200), Some(1..100));
assert_eq!(bc.needed_blocks(peer0.clone(), 128, 10000, 600, 1, 200), None); // too far ahead
assert_eq!(bc.needed_blocks(peer0.clone(), 128, 10000, 600, 1, 200000), Some(100 + 128 .. 100 + 128 + 128));
assert_eq!(
bc.needed_blocks(peer0.clone(), 128, 10000, 600, 1, 200000),
Some(100 + 128..100 + 128 + 128)
);
}
}
@@ -16,14 +16,16 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use sp_blockchain::Error as ClientError;
use crate::protocol::sync::{PeerSync, PeerSyncState};
use fork_tree::ForkTree;
use libp2p::PeerId;
use log::{debug, trace, warn};
use sp_blockchain::Error as ClientError;
use sp_runtime::traits::{Block as BlockT, NumberFor, Zero};
use std::collections::{HashMap, HashSet, VecDeque};
use std::time::Duration;
use std::{
collections::{HashMap, HashSet, VecDeque},
time::Duration,
};
use wasm_timer::Instant;
// Time to wait before trying to get the same extra data from the same peer.
@@ -61,7 +63,7 @@ pub(crate) struct Metrics {
pub(crate) active_requests: u32,
pub(crate) importing_requests: u32,
pub(crate) failed_requests: u32,
_priv: ()
_priv: (),
}
impl<B: BlockT> ExtraRequests<B> {
@@ -93,13 +95,14 @@ impl<B: BlockT> ExtraRequests<B> {
/// Queue an extra data request to be considered by the `Matcher`.
pub(crate) fn schedule<F>(&mut self, request: ExtraRequest<B>, is_descendent_of: F)
where F: Fn(&B::Hash, &B::Hash) -> Result<bool, ClientError>
where
F: Fn(&B::Hash, &B::Hash) -> Result<bool, ClientError>,
{
match self.tree.import(request.0, request.1, (), &is_descendent_of) {
Ok(true) => {
// this is a new root so we add it to the current `pending_requests`
self.pending_requests.push_back((request.0, request.1));
}
},
Err(fork_tree::Error::Revert) => {
// we have finalized further than the given request, presumably
// by some other part of the system (not sync). we can safely
@@ -107,8 +110,8 @@ impl<B: BlockT> ExtraRequests<B> {
},
Err(err) => {
debug!(target: "sync", "Failed to insert request {:?} into tree: {:?}", request, err);
}
_ => ()
},
_ => (),
}
}
@@ -120,7 +123,11 @@ impl<B: BlockT> ExtraRequests<B> {
}
/// Processes the response for the request previously sent to the given peer.
pub(crate) fn on_response<R>(&mut self, who: PeerId, resp: Option<R>) -> Option<(PeerId, B::Hash, NumberFor<B>, R)> {
pub(crate) fn on_response<R>(
&mut self,
who: PeerId,
resp: Option<R>,
) -> Option<(PeerId, B::Hash, NumberFor<B>, R)> {
// we assume that the request maps to the given response, this is
// currently enforced by the outer network protocol before passing on
// messages to chain sync.
@@ -157,9 +164,10 @@ impl<B: BlockT> ExtraRequests<B> {
&mut self,
best_finalized_hash: &B::Hash,
best_finalized_number: NumberFor<B>,
is_descendent_of: F
is_descendent_of: F,
) -> Result<(), fork_tree::Error<ClientError>>
where F: Fn(&B::Hash, &B::Hash) -> Result<bool, ClientError>
where
F: Fn(&B::Hash, &B::Hash) -> Result<bool, ClientError>,
{
let request = (*best_finalized_hash, best_finalized_number);
@@ -203,9 +211,8 @@ impl<B: BlockT> ExtraRequests<B> {
&mut self,
request: ExtraRequest<B>,
result: Result<ExtraRequest<B>, E>,
reschedule_on_failure: bool
) -> bool
{
reschedule_on_failure: bool,
) -> bool {
if !self.importing_requests.remove(&request) {
return false
}
@@ -217,7 +224,7 @@ impl<B: BlockT> ExtraRequests<B> {
self.pending_requests.push_front(request);
}
return true
}
},
};
if self.tree.finalize_root(&finalized_hash).is_none() {
@@ -258,7 +265,7 @@ impl<B: BlockT> ExtraRequests<B> {
active_requests: self.active_requests.len().try_into().unwrap_or(std::u32::MAX),
failed_requests: self.failed_requests.len().try_into().unwrap_or(std::u32::MAX),
importing_requests: self.importing_requests.len().try_into().unwrap_or(std::u32::MAX),
_priv: ()
_priv: (),
}
}
}
@@ -269,15 +276,12 @@ pub(crate) struct Matcher<'a, B: BlockT> {
/// Length of pending requests collection.
/// Used to ensure we do not loop more than once over all pending requests.
remaining: usize,
extras: &'a mut ExtraRequests<B>
extras: &'a mut ExtraRequests<B>,
}
impl<'a, B: BlockT> Matcher<'a, B> {
fn new(extras: &'a mut ExtraRequests<B>) -> Self {
Matcher {
remaining: extras.pending_requests.len(),
extras
}
Matcher { remaining: extras.pending_requests.len(), extras }
}
/// Finds a peer to which a pending request can be sent.
@@ -294,7 +298,10 @@ impl<'a, B: BlockT> Matcher<'a, B> {
///
/// The returned `PeerId` (if any) is guaranteed to come from the given `peers`
/// argument.
pub(crate) fn next(&mut self, peers: &HashMap<PeerId, PeerSync<B>>) -> Option<(PeerId, ExtraRequest<B>)> {
pub(crate) fn next(
&mut self,
peers: &HashMap<PeerId, PeerSync<B>>,
) -> Option<(PeerId, ExtraRequest<B>)> {
if self.remaining == 0 {
return None
}
@@ -305,7 +312,9 @@ impl<'a, B: BlockT> Matcher<'a, B> {
}
while let Some(request) = self.extras.pending_requests.pop_front() {
for (peer, sync) in peers.iter().filter(|(_, sync)| sync.state == PeerSyncState::Available) {
for (peer, sync) in
peers.iter().filter(|(_, sync)| sync.state == PeerSyncState::Available)
{
// only ask peers that have synced at least up to the block number that we're asking the extra for
if sync.best_number < request.1 {
continue
@@ -315,7 +324,13 @@ impl<'a, B: BlockT> Matcher<'a, B> {
continue
}
// only ask if the same request has not failed for this peer before
if self.extras.failed_requests.get(&request).map(|rr| rr.iter().any(|i| &i.0 == peer)).unwrap_or(false) {
if self
.extras
.failed_requests
.get(&request)
.map(|rr| rr.iter().any(|i| &i.0 == peer))
.unwrap_or(false)
{
continue
}
self.extras.active_requests.insert(peer.clone(), request);
@@ -343,22 +358,22 @@ impl<'a, B: BlockT> Matcher<'a, B> {
#[cfg(test)]
mod tests {
use crate::protocol::sync::PeerSync;
use sp_blockchain::Error as ClientError;
use quickcheck::{Arbitrary, Gen, QuickCheck};
use std::collections::{HashMap, HashSet};
use super::*;
use crate::protocol::sync::PeerSync;
use quickcheck::{Arbitrary, Gen, QuickCheck};
use sp_blockchain::Error as ClientError;
use sp_test_primitives::{Block, BlockNumber, Hash};
use std::collections::{HashMap, HashSet};
#[test]
fn requests_are_processed_in_order() {
fn property(mut peers: ArbitraryPeers) {
let mut requests = ExtraRequests::<Block>::new("test");
let num_peers_available = peers.0.values()
.filter(|s| s.state == PeerSyncState::Available).count();
let num_peers_available =
peers.0.values().filter(|s| s.state == PeerSyncState::Available).count();
for i in 0 .. num_peers_available {
for i in 0..num_peers_available {
requests.schedule((Hash::random(), i as u64), |a, b| Ok(a[0] >= b[0]))
}
@@ -368,12 +383,12 @@ mod tests {
for p in &pending {
let (peer, r) = m.next(&peers.0).unwrap();
assert_eq!(p, &r);
peers.0.get_mut(&peer).unwrap().state = PeerSyncState::DownloadingJustification(r.0);
peers.0.get_mut(&peer).unwrap().state =
PeerSyncState::DownloadingJustification(r.0);
}
}
QuickCheck::new()
.quickcheck(property as fn(ArbitraryPeers))
QuickCheck::new().quickcheck(property as fn(ArbitraryPeers))
}
#[test]
@@ -398,22 +413,24 @@ mod tests {
fn property(mut peers: ArbitraryPeers) -> bool {
let mut requests = ExtraRequests::<Block>::new("test");
let num_peers_available = peers.0.values()
.filter(|s| s.state == PeerSyncState::Available).count();
let num_peers_available =
peers.0.values().filter(|s| s.state == PeerSyncState::Available).count();
for i in 0 .. num_peers_available {
for i in 0..num_peers_available {
requests.schedule((Hash::random(), i as u64), |a, b| Ok(a[0] >= b[0]))
}
let mut m = requests.matcher();
while let Some((peer, r)) = m.next(&peers.0) {
peers.0.get_mut(&peer).unwrap().state = PeerSyncState::DownloadingJustification(r.0);
peers.0.get_mut(&peer).unwrap().state =
PeerSyncState::DownloadingJustification(r.0);
}
assert!(requests.pending_requests.is_empty());
let active_peers = requests.active_requests.keys().cloned().collect::<Vec<_>>();
let previously_active = requests.active_requests.values().cloned().collect::<HashSet<_>>();
let previously_active =
requests.active_requests.values().cloned().collect::<HashSet<_>>();
for peer in &active_peers {
requests.peer_disconnected(peer)
@@ -424,8 +441,7 @@ mod tests {
previously_active == requests.pending_requests.iter().cloned().collect::<HashSet<_>>()
}
QuickCheck::new()
.quickcheck(property as fn(ArbitraryPeers) -> bool)
QuickCheck::new().quickcheck(property as fn(ArbitraryPeers) -> bool)
}
#[test]
@@ -433,31 +449,44 @@ mod tests {
fn property(mut peers: ArbitraryPeers) {
let mut requests = ExtraRequests::<Block>::new("test");
let num_peers_available = peers.0.values()
.filter(|s| s.state == PeerSyncState::Available).count();
let num_peers_available =
peers.0.values().filter(|s| s.state == PeerSyncState::Available).count();
for i in 0 .. num_peers_available {
for i in 0..num_peers_available {
requests.schedule((Hash::random(), i as u64), |a, b| Ok(a[0] >= b[0]))
}
let mut m = requests.matcher();
while let Some((peer, r)) = m.next(&peers.0) {
peers.0.get_mut(&peer).unwrap().state = PeerSyncState::DownloadingJustification(r.0);
peers.0.get_mut(&peer).unwrap().state =
PeerSyncState::DownloadingJustification(r.0);
}
let active = requests.active_requests.iter().map(|(p, &r)| (p.clone(), r)).collect::<Vec<_>>();
let active = requests
.active_requests
.iter()
.map(|(p, &r)| (p.clone(), r))
.collect::<Vec<_>>();
for (peer, req) in &active {
assert!(requests.failed_requests.get(req).is_none());
assert!(!requests.pending_requests.contains(req));
assert!(requests.on_response::<()>(peer.clone(), None).is_none());
assert!(requests.pending_requests.contains(req));
assert_eq!(1, requests.failed_requests.get(req).unwrap().iter().filter(|(p, _)| p == peer).count())
assert_eq!(
1,
requests
.failed_requests
.get(req)
.unwrap()
.iter()
.filter(|(p, _)| p == peer)
.count()
)
}
}
QuickCheck::new()
.quickcheck(property as fn(ArbitraryPeers))
QuickCheck::new().quickcheck(property as fn(ArbitraryPeers))
}
#[test]
@@ -497,7 +526,10 @@ mod tests {
finality_proofs.try_finalize_root::<()>((hash6, 6), Ok((hash7, 7)), true);
// ensure that there's no request for #6
assert_eq!(finality_proofs.pending_requests.iter().collect::<Vec<_>>(), Vec::<&(Hash, u64)>::new());
assert_eq!(
finality_proofs.pending_requests.iter().collect::<Vec<_>>(),
Vec::<&(Hash, u64)>::new()
);
}
#[test]
@@ -560,7 +592,7 @@ mod tests {
impl Arbitrary for ArbitraryPeers {
fn arbitrary(g: &mut Gen) -> Self {
let mut peers = HashMap::with_capacity(g.size());
for _ in 0 .. g.size() {
for _ in 0..g.size() {
let ps = ArbitraryPeerSync::arbitrary(g).0;
peers.insert(ps.peer_id.clone(), ps);
}
@@ -16,13 +16,15 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use std::sync::Arc;
use codec::{Encode, Decode};
use sp_runtime::traits::{Block as BlockT, Header, NumberFor};
use sc_client_api::StorageProof;
use crate::schema::v1::{StateRequest, StateResponse, StateEntry};
use crate::chain::{Client, ImportedState};
use super::StateDownloadProgress;
use crate::{
chain::{Client, ImportedState},
schema::v1::{StateEntry, StateRequest, StateResponse},
};
use codec::{Decode, Encode};
use sc_client_api::StorageProof;
use sp_runtime::traits::{Block as BlockT, Header, NumberFor};
use std::sync::Arc;
/// State sync support.
@@ -73,14 +75,14 @@ impl<B: BlockT> StateSync<B> {
target: "sync",
"Bad state response",
);
return ImportResult::BadResponse;
return ImportResult::BadResponse
}
if !self.skip_proof && response.proof.is_empty() {
log::debug!(
target: "sync",
"Missing proof",
);
return ImportResult::BadResponse;
return ImportResult::BadResponse
}
let complete = if !self.skip_proof {
log::debug!(
@@ -93,24 +95,21 @@ impl<B: BlockT> StateSync<B> {
Ok(proof) => proof,
Err(e) => {
log::debug!(target: "sync", "Error decoding proof: {:?}", e);
return ImportResult::BadResponse;
}
};
let (values, complete) = match self.client.verify_range_proof(
self.target_root,
proof,
&self.last_key
) {
Err(e) => {
log::debug!(
target: "sync",
"StateResponse failed proof verification: {:?}",
e,
);
return ImportResult::BadResponse;
return ImportResult::BadResponse
},
Ok(values) => values,
};
let (values, complete) =
match self.client.verify_range_proof(self.target_root, proof, &self.last_key) {
Err(e) => {
log::debug!(
target: "sync",
"StateResponse failed proof verification: {:?}",
e,
);
return ImportResult::BadResponse
},
Ok(values) => values,
};
log::debug!(target: "sync", "Imported with {} keys", values.len());
if let Some(last) = values.last().map(|(k, _)| k) {
@@ -120,7 +119,7 @@ impl<B: BlockT> StateSync<B> {
for (key, value) in values {
self.imported_bytes += key.len() as u64;
self.state.push((key, value))
};
}
self.imported_bytes += proof_size;
complete
} else {
@@ -142,10 +141,14 @@ impl<B: BlockT> StateSync<B> {
};
if complete {
self.complete = true;
ImportResult::Import(self.target_block.clone(), self.target_header.clone(), ImportedState {
block: self.target_block.clone(),
state: std::mem::take(&mut self.state)
})
ImportResult::Import(
self.target_block.clone(),
self.target_header.clone(),
ImportedState {
block: self.target_block.clone(),
state: std::mem::take(&mut self.state),
},
)
} else {
ImportResult::Continue(self.next_request())
}
@@ -178,10 +181,6 @@ impl<B: BlockT> StateSync<B> {
/// Returns state sync estimated progress.
pub fn progress(&self) -> StateDownloadProgress {
let percent_done = (*self.last_key.get(0).unwrap_or(&0u8) as u32) * 100 / 256;
StateDownloadProgress {
percentage: percent_done,
size: self.imported_bytes,
}
StateDownloadProgress { percentage: percent_done, size: self.imported_bytes }
}
}
+334 -302
View File
@@ -33,17 +33,20 @@
//!
//! - If provided, a ["requests processing"](ProtocolConfig::inbound_queue) channel
//! is used to handle incoming requests.
//!
use futures::{channel::{mpsc, oneshot}, prelude::*};
use crate::ReputationChange;
use futures::{
channel::{mpsc, oneshot},
prelude::*,
};
use libp2p::{
core::{
connection::{ConnectionId, ListenerId},
ConnectedPoint, Multiaddr, PeerId,
},
request_response::{
RequestResponse, RequestResponseCodec, RequestResponseConfig, RequestResponseEvent,
RequestResponseMessage, ResponseChannel, ProtocolSupport
ProtocolSupport, RequestResponse, RequestResponseCodec, RequestResponseConfig,
RequestResponseEvent, RequestResponseMessage, ResponseChannel,
},
swarm::{
protocols_handler::multi::MultiHandler, NetworkBehaviour, NetworkBehaviourAction,
@@ -51,58 +54,62 @@ use libp2p::{
},
};
use std::{
borrow::Cow, collections::{hash_map::Entry, HashMap}, convert::TryFrom as _, io, iter,
pin::Pin, task::{Context, Poll}, time::Duration,
borrow::Cow,
collections::{hash_map::Entry, HashMap},
convert::TryFrom as _,
io, iter,
pin::Pin,
task::{Context, Poll},
time::Duration,
};
use wasm_timer::Instant;
use crate::ReputationChange;
pub use libp2p::request_response::{InboundFailure, OutboundFailure, RequestId};
/// Configuration for a single request-response protocol.
#[derive(Debug, Clone)]
pub struct ProtocolConfig {
/// Name of the protocol on the wire. Should be something like `/foo/bar`.
pub name: Cow<'static, str>,
/// Name of the protocol on the wire. Should be something like `/foo/bar`.
pub name: Cow<'static, str>,
/// Maximum allowed size, in bytes, of a request.
///
/// Any request larger than this value will be declined as a way to avoid allocating too
/// much memory for it.
pub max_request_size: u64,
/// Maximum allowed size, in bytes, of a request.
///
/// Any request larger than this value will be declined as a way to avoid allocating too
/// much memory for it.
pub max_request_size: u64,
/// Maximum allowed size, in bytes, of a response.
///
/// Any response larger than this value will be declined as a way to avoid allocating too
/// much memory for it.
pub max_response_size: u64,
/// Maximum allowed size, in bytes, of a response.
///
/// Any response larger than this value will be declined as a way to avoid allocating too
/// much memory for it.
pub max_response_size: u64,
/// Duration after which emitted requests are considered timed out.
///
/// If you expect the response to come back quickly, you should set this to a smaller duration.
pub request_timeout: Duration,
/// Duration after which emitted requests are considered timed out.
///
/// If you expect the response to come back quickly, you should set this to a smaller duration.
pub request_timeout: Duration,
/// Channel on which the networking service will send incoming requests.
///
/// Every time a peer sends a request to the local node using this protocol, the networking
/// service will push an element on this channel. The receiving side of this channel then has
/// to pull this element, process the request, and send back the response to send back to the
/// peer.
///
/// The size of the channel has to be carefully chosen. If the channel is full, the networking
/// service will discard the incoming request send back an error to the peer. Consequently,
/// the channel being full is an indicator that the node is overloaded.
///
/// You can typically set the size of the channel to `T / d`, where `T` is the
/// `request_timeout` and `d` is the expected average duration of CPU and I/O it takes to
/// build a response.
///
/// Can be `None` if the local node does not support answering incoming requests.
/// If this is `None`, then the local node will not advertise support for this protocol towards
/// other peers. If this is `Some` but the channel is closed, then the local node will
/// advertise support for this protocol, but any incoming request will lead to an error being
/// sent back.
pub inbound_queue: Option<mpsc::Sender<IncomingRequest>>,
/// Channel on which the networking service will send incoming requests.
///
/// Every time a peer sends a request to the local node using this protocol, the networking
/// service will push an element on this channel. The receiving side of this channel then has
/// to pull this element, process the request, and send back the response to send back to the
/// peer.
///
/// The size of the channel has to be carefully chosen. If the channel is full, the networking
/// service will discard the incoming request send back an error to the peer. Consequently,
/// the channel being full is an indicator that the node is overloaded.
///
/// You can typically set the size of the channel to `T / d`, where `T` is the
/// `request_timeout` and `d` is the expected average duration of CPU and I/O it takes to
/// build a response.
///
/// Can be `None` if the local node does not support answering incoming requests.
/// If this is `None`, then the local node will not advertise support for this protocol towards
/// other peers. If this is `Some` but the channel is closed, then the local node will
/// advertise support for this protocol, but any incoming request will lead to an error being
/// sent back.
pub inbound_queue: Option<mpsc::Sender<IncomingRequest>>,
}
/// A single request received by a peer on a request-response protocol.
@@ -179,14 +186,11 @@ pub enum Event {
/// Duration the request took.
duration: Duration,
/// Result of the request.
result: Result<(), RequestFailure>
result: Result<(), RequestFailure>,
},
/// A request protocol handler issued reputation changes for the given peer.
ReputationChanges {
peer: PeerId,
changes: Vec<ReputationChange>,
}
ReputationChanges { peer: PeerId, changes: Vec<ReputationChange> },
}
/// Combination of a protocol name and a request id.
@@ -234,19 +238,17 @@ pub struct RequestResponsesBehaviour {
/// "response builder" used to build responses for incoming requests.
protocols: HashMap<
Cow<'static, str>,
(RequestResponse<GenericCodec>, Option<mpsc::Sender<IncomingRequest>>)
(RequestResponse<GenericCodec>, Option<mpsc::Sender<IncomingRequest>>),
>,
/// Pending requests, passed down to a [`RequestResponse`] behaviour, awaiting a reply.
pending_requests: HashMap<
ProtocolRequestId,
(Instant, oneshot::Sender<Result<Vec<u8>, RequestFailure>>),
>,
pending_requests:
HashMap<ProtocolRequestId, (Instant, oneshot::Sender<Result<Vec<u8>, RequestFailure>>)>,
/// Whenever an incoming request arrives, a `Future` is added to this list and will yield the
/// start time and the response to send back to the remote.
pending_responses: stream::FuturesUnordered<
Pin<Box<dyn Future<Output = Option<RequestProcessingOutcome>> + Send>>
Pin<Box<dyn Future<Output = Option<RequestProcessingOutcome>> + Send>>,
>,
/// Whenever an incoming request arrives, the arrival [`Instant`] is recorded here.
@@ -282,15 +284,18 @@ impl RequestResponsesBehaviour {
ProtocolSupport::Outbound
};
let rq_rp = RequestResponse::new(GenericCodec {
max_request_size: protocol.max_request_size,
max_response_size: protocol.max_response_size,
}, iter::once((protocol.name.as_bytes().to_vec(), protocol_support)), cfg);
let rq_rp = RequestResponse::new(
GenericCodec {
max_request_size: protocol.max_request_size,
max_response_size: protocol.max_response_size,
},
iter::once((protocol.name.as_bytes().to_vec(), protocol_support)),
cfg,
);
match protocols.entry(protocol.name) {
Entry::Vacant(e) => e.insert((rq_rp, protocol.inbound_queue)),
Entry::Occupied(e) =>
return Err(RegisterError::DuplicateProtocol(e.key().clone())),
Entry::Occupied(e) => return Err(RegisterError::DuplicateProtocol(e.key().clone())),
};
}
@@ -348,19 +353,20 @@ impl RequestResponsesBehaviour {
}
impl NetworkBehaviour for RequestResponsesBehaviour {
type ProtocolsHandler = MultiHandler<
String,
<RequestResponse<GenericCodec> as NetworkBehaviour>::ProtocolsHandler,
>;
type ProtocolsHandler =
MultiHandler<String, <RequestResponse<GenericCodec> as NetworkBehaviour>::ProtocolsHandler>;
type OutEvent = Event;
fn new_handler(&mut self) -> Self::ProtocolsHandler {
let iter = self.protocols.iter_mut()
let iter = self
.protocols
.iter_mut()
.map(|(p, (r, _))| (p.to_string(), NetworkBehaviour::new_handler(r)));
MultiHandler::try_from_iter(iter)
.expect("Protocols are in a HashMap and there can be at most one handler per \
protocol name, which is the only possible error; qed")
MultiHandler::try_from_iter(iter).expect(
"Protocols are in a HashMap and there can be at most one handler per \
protocol name, which is the only possible error; qed",
)
}
fn addresses_of_peer(&mut self, _: &PeerId) -> Vec<Multiaddr> {
@@ -384,7 +390,12 @@ impl NetworkBehaviour for RequestResponsesBehaviour {
}
}
fn inject_connection_closed(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) {
fn inject_connection_closed(
&mut self,
peer_id: &PeerId,
conn: &ConnectionId,
endpoint: &ConnectedPoint,
) {
for (p, _) in self.protocols.values_mut() {
NetworkBehaviour::inject_connection_closed(p, peer_id, conn, endpoint)
}
@@ -400,7 +411,7 @@ impl NetworkBehaviour for RequestResponsesBehaviour {
&mut self,
peer_id: Option<&PeerId>,
addr: &Multiaddr,
error: &dyn std::error::Error
error: &dyn std::error::Error,
) {
for (p, _) in self.protocols.values_mut() {
NetworkBehaviour::inject_addr_reach_failure(p, peer_id, addr, error)
@@ -488,11 +499,7 @@ impl NetworkBehaviour for RequestResponsesBehaviour {
request_id,
protocol: protocol_name,
inner_channel,
response: OutgoingResponse {
result,
reputation_changes,
sent_feedback,
},
response: OutgoingResponse { result, reputation_changes, sent_feedback },
} = match outcome {
Some(outcome) => outcome,
// The response builder was too busy or handling the request failed. This is
@@ -514,10 +521,8 @@ impl NetworkBehaviour for RequestResponsesBehaviour {
);
} else {
if let Some(sent_feedback) = sent_feedback {
self.send_feedback.insert(
(protocol_name, request_id).into(),
sent_feedback
);
self.send_feedback
.insert((protocol_name, request_id).into(), sent_feedback);
}
}
}
@@ -525,11 +530,8 @@ impl NetworkBehaviour for RequestResponsesBehaviour {
if !reputation_changes.is_empty() {
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(
Event::ReputationChanges{
peer,
changes: reputation_changes,
},
));
Event::ReputationChanges { peer, changes: reputation_changes },
))
}
}
@@ -543,38 +545,35 @@ impl NetworkBehaviour for RequestResponsesBehaviour {
// Other events generated by the underlying behaviour are transparently
// passed through.
NetworkBehaviourAction::DialAddress { address } => {
log::error!("The request-response isn't supposed to start dialing peers");
log::error!(
"The request-response isn't supposed to start dialing peers"
);
return Poll::Ready(NetworkBehaviourAction::DialAddress { address })
}
NetworkBehaviourAction::DialPeer { peer_id, condition } => {
},
NetworkBehaviourAction::DialPeer { peer_id, condition } =>
return Poll::Ready(NetworkBehaviourAction::DialPeer {
peer_id,
condition,
})
}
NetworkBehaviourAction::NotifyHandler {
peer_id,
handler,
event,
} => {
}),
NetworkBehaviourAction::NotifyHandler { peer_id, handler, event } =>
return Poll::Ready(NetworkBehaviourAction::NotifyHandler {
peer_id,
handler,
event: ((*protocol).to_string(), event),
})
}
NetworkBehaviourAction::ReportObservedAddr { address, score } => {
}),
NetworkBehaviourAction::ReportObservedAddr { address, score } =>
return Poll::Ready(NetworkBehaviourAction::ReportObservedAddr {
address, score,
})
}
address,
score,
}),
};
match ev {
// Received a request from a remote.
RequestResponseEvent::Message {
peer,
message: RequestResponseMessage::Request { request_id, request, channel, .. },
message:
RequestResponseMessage::Request { request_id, request, channel, .. },
} => {
self.pending_responses_arrival_time.insert(
(protocol.clone(), request_id.clone()).into(),
@@ -605,7 +604,11 @@ impl NetworkBehaviour for RequestResponsesBehaviour {
// `InboundFailure::Omission` event.
if let Ok(response) = rx.await {
Some(RequestProcessingOutcome {
peer, request_id, protocol, inner_channel: channel, response
peer,
request_id,
protocol,
inner_channel: channel,
response,
})
} else {
None
@@ -614,27 +617,25 @@ impl NetworkBehaviour for RequestResponsesBehaviour {
// This `continue` makes sure that `pending_responses` gets polled
// after we have added the new element.
continue 'poll_all;
}
continue 'poll_all
},
// Received a response from a remote to one of our requests.
RequestResponseEvent::Message {
peer,
message: RequestResponseMessage::Response {
request_id,
response,
},
message: RequestResponseMessage::Response { request_id, response },
..
} => {
let (started, delivered) = match self.pending_requests.remove(
&(protocol.clone(), request_id).into(),
) {
let (started, delivered) = match self
.pending_requests
.remove(&(protocol.clone(), request_id).into())
{
Some((started, pending_response)) => {
let delivered = pending_response.send(
response.map_err(|()| RequestFailure::Refused),
).map_err(|_| RequestFailure::Obsolete);
let delivered = pending_response
.send(response.map_err(|()| RequestFailure::Refused))
.map_err(|_| RequestFailure::Obsolete);
(started, delivered)
}
},
None => {
log::warn!(
target: "sub-libp2p",
@@ -642,8 +643,8 @@ impl NetworkBehaviour for RequestResponsesBehaviour {
request_id,
);
debug_assert!(false);
continue;
}
continue
},
};
let out = Event::RequestFinished {
@@ -653,21 +654,22 @@ impl NetworkBehaviour for RequestResponsesBehaviour {
result: delivered,
};
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(out));
}
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(out))
},
// One of our requests has failed.
RequestResponseEvent::OutboundFailure {
peer,
request_id,
error,
..
peer, request_id, error, ..
} => {
let started = match self.pending_requests.remove(&(protocol.clone(), request_id).into()) {
let started = match self
.pending_requests
.remove(&(protocol.clone(), request_id).into())
{
Some((started, pending_response)) => {
if pending_response.send(
Err(RequestFailure::Network(error.clone())),
).is_err() {
if pending_response
.send(Err(RequestFailure::Network(error.clone())))
.is_err()
{
log::debug!(
target: "sub-libp2p",
"Request with id {:?} failed. At the same time local \
@@ -676,7 +678,7 @@ impl NetworkBehaviour for RequestResponsesBehaviour {
);
}
started
}
},
None => {
log::warn!(
target: "sub-libp2p",
@@ -684,8 +686,8 @@ impl NetworkBehaviour for RequestResponsesBehaviour {
request_id,
);
debug_assert!(false);
continue;
}
continue
},
};
let out = Event::RequestFinished {
@@ -695,29 +697,30 @@ impl NetworkBehaviour for RequestResponsesBehaviour {
result: Err(RequestFailure::Network(error)),
};
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(out));
}
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(out))
},
// An inbound request failed, either while reading the request or due to failing
// to send a response.
RequestResponseEvent::InboundFailure { request_id, peer, error, .. } => {
self.pending_responses_arrival_time.remove(
&(protocol.clone(), request_id).into(),
);
RequestResponseEvent::InboundFailure {
request_id, peer, error, ..
} => {
self.pending_responses_arrival_time
.remove(&(protocol.clone(), request_id).into());
self.send_feedback.remove(&(protocol.clone(), request_id).into());
let out = Event::InboundRequest {
peer,
protocol: protocol.clone(),
result: Err(ResponseFailure::Network(error)),
};
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(out));
}
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(out))
},
// A response to an inbound request has been sent.
RequestResponseEvent::ResponseSent { request_id, peer } => {
let arrival_time = self.pending_responses_arrival_time.remove(
&(protocol.clone(), request_id).into(),
)
let arrival_time = self
.pending_responses_arrival_time
.remove(&(protocol.clone(), request_id).into())
.map(|t| t.elapsed())
.expect(
"Time is added for each inbound request on arrival and only \
@@ -727,9 +730,9 @@ impl NetworkBehaviour for RequestResponsesBehaviour {
failed; qed.",
);
if let Some(send_feedback) = self.send_feedback.remove(
&(protocol.clone(), request_id).into()
) {
if let Some(send_feedback) =
self.send_feedback.remove(&(protocol.clone(), request_id).into())
{
let _ = send_feedback.send(());
}
@@ -739,14 +742,13 @@ impl NetworkBehaviour for RequestResponsesBehaviour {
result: Ok(arrival_time),
};
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(out));
}
return Poll::Ready(NetworkBehaviourAction::GenerateEvent(out))
},
};
}
}
break Poll::Pending;
break Poll::Pending
}
}
}
@@ -786,7 +788,7 @@ pub enum ResponseFailure {
/// Implements the libp2p [`RequestResponseCodec`] trait. Defines how streams of bytes are turned
/// into requests and responses and vice-versa.
#[derive(Debug, Clone)]
#[doc(hidden)] // Needs to be public in order to satisfy the Rust compiler.
#[doc(hidden)] // Needs to be public in order to satisfy the Rust compiler.
pub struct GenericCodec {
max_request_size: u64,
max_response_size: u64,
@@ -807,13 +809,14 @@ impl RequestResponseCodec for GenericCodec {
T: AsyncRead + Unpin + Send,
{
// Read the length.
let length = unsigned_varint::aio::read_usize(&mut io).await
let length = unsigned_varint::aio::read_usize(&mut io)
.await
.map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err))?;
if length > usize::try_from(self.max_request_size).unwrap_or(usize::MAX) {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!("Request size exceeds limit: {} > {}", length, self.max_request_size)
));
format!("Request size exceeds limit: {} > {}", length, self.max_request_size),
))
}
// Read the payload.
@@ -840,17 +843,15 @@ impl RequestResponseCodec for GenericCodec {
Ok(l) => l,
Err(unsigned_varint::io::ReadError::Io(err))
if matches!(err.kind(), io::ErrorKind::UnexpectedEof) =>
{
return Ok(Err(()));
}
return Ok(Err(())),
Err(err) => return Err(io::Error::new(io::ErrorKind::InvalidInput, err)),
};
if length > usize::try_from(self.max_response_size).unwrap_or(usize::MAX) {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!("Response size exceeds limit: {} > {}", length, self.max_response_size)
));
format!("Response size exceeds limit: {} > {}", length, self.max_response_size),
))
}
// Read the payload.
@@ -913,23 +914,30 @@ impl RequestResponseCodec for GenericCodec {
mod tests {
use super::*;
use futures::channel::{mpsc, oneshot};
use futures::executor::LocalPool;
use futures::task::Spawn;
use libp2p::identity::Keypair;
use libp2p::Multiaddr;
use libp2p::core::upgrade;
use libp2p::core::transport::{Transport, MemoryTransport};
use libp2p::noise;
use libp2p::swarm::{Swarm, SwarmEvent};
use futures::{
channel::{mpsc, oneshot},
executor::LocalPool,
task::Spawn,
};
use libp2p::{
core::{
transport::{MemoryTransport, Transport},
upgrade,
},
identity::Keypair,
noise,
swarm::{Swarm, SwarmEvent},
Multiaddr,
};
use std::{iter, time::Duration};
fn build_swarm(list: impl Iterator<Item = ProtocolConfig>) -> (Swarm<RequestResponsesBehaviour>, Multiaddr) {
fn build_swarm(
list: impl Iterator<Item = ProtocolConfig>,
) -> (Swarm<RequestResponsesBehaviour>, Multiaddr) {
let keypair = Keypair::generate_ed25519();
let noise_keys = noise::Keypair::<noise::X25519Spec>::new()
.into_authentic(&keypair)
.unwrap();
let noise_keys =
noise::Keypair::<noise::X25519Spec>::new().into_authentic(&keypair).unwrap();
let transport = MemoryTransport
.upgrade(upgrade::Version::V1)
@@ -956,18 +964,24 @@ mod tests {
.map(|_| {
let (tx, mut rx) = mpsc::channel::<IncomingRequest>(64);
pool.spawner().spawn_obj(async move {
while let Some(rq) = rx.next().await {
let (fb_tx, fb_rx) = oneshot::channel();
assert_eq!(rq.payload, b"this is a request");
let _ = rq.pending_response.send(super::OutgoingResponse {
result: Ok(b"this is a response".to_vec()),
reputation_changes: Vec::new(),
sent_feedback: Some(fb_tx),
});
fb_rx.await.unwrap();
}
}.boxed().into()).unwrap();
pool.spawner()
.spawn_obj(
async move {
while let Some(rq) = rx.next().await {
let (fb_tx, fb_rx) = oneshot::channel();
assert_eq!(rq.payload, b"this is a request");
let _ = rq.pending_response.send(super::OutgoingResponse {
result: Ok(b"this is a response".to_vec()),
reputation_changes: Vec::new(),
sent_feedback: Some(fb_tx),
});
fb_rx.await.unwrap();
}
}
.boxed()
.into(),
)
.unwrap();
let protocol_config = ProtocolConfig {
name: From::from(protocol_name),
@@ -989,19 +1003,23 @@ mod tests {
}
// Running `swarm[0]` in the background.
pool.spawner().spawn_obj({
let (mut swarm, _) = swarms.remove(0);
async move {
loop {
match swarm.next_event().await {
SwarmEvent::Behaviour(Event::InboundRequest { result, .. }) => {
result.unwrap();
},
_ => {}
pool.spawner()
.spawn_obj({
let (mut swarm, _) = swarms.remove(0);
async move {
loop {
match swarm.next_event().await {
SwarmEvent::Behaviour(Event::InboundRequest { result, .. }) => {
result.unwrap();
},
_ => {},
}
}
}
}.boxed().into()
}).unwrap();
.boxed()
.into()
})
.unwrap();
// Remove and run the remaining swarm.
let (mut swarm, _) = swarms.remove(0);
@@ -1021,14 +1039,12 @@ mod tests {
);
assert!(response_receiver.is_none());
response_receiver = Some(receiver);
}
SwarmEvent::Behaviour(Event::RequestFinished {
result, ..
}) => {
},
SwarmEvent::Behaviour(Event::RequestFinished { result, .. }) => {
result.unwrap();
break;
}
_ => {}
break
},
_ => {},
}
}
@@ -1046,21 +1062,27 @@ mod tests {
.map(|_| {
let (tx, mut rx) = mpsc::channel::<IncomingRequest>(64);
pool.spawner().spawn_obj(async move {
while let Some(rq) = rx.next().await {
assert_eq!(rq.payload, b"this is a request");
let _ = rq.pending_response.send(super::OutgoingResponse {
result: Ok(b"this response exceeds the limit".to_vec()),
reputation_changes: Vec::new(),
sent_feedback: None,
});
}
}.boxed().into()).unwrap();
pool.spawner()
.spawn_obj(
async move {
while let Some(rq) = rx.next().await {
assert_eq!(rq.payload, b"this is a request");
let _ = rq.pending_response.send(super::OutgoingResponse {
result: Ok(b"this response exceeds the limit".to_vec()),
reputation_changes: Vec::new(),
sent_feedback: None,
});
}
}
.boxed()
.into(),
)
.unwrap();
let protocol_config = ProtocolConfig {
name: From::from(protocol_name),
max_request_size: 1024,
max_response_size: 8, // <-- important for the test
max_response_size: 8, // <-- important for the test
request_timeout: Duration::from_secs(30),
inbound_queue: Some(tx),
};
@@ -1078,20 +1100,24 @@ mod tests {
// Running `swarm[0]` in the background until a `InboundRequest` event happens,
// which is a hint about the test having ended.
pool.spawner().spawn_obj({
let (mut swarm, _) = swarms.remove(0);
async move {
loop {
match swarm.next_event().await {
SwarmEvent::Behaviour(Event::InboundRequest { result, .. }) => {
assert!(result.is_ok());
break
},
_ => {}
pool.spawner()
.spawn_obj({
let (mut swarm, _) = swarms.remove(0);
async move {
loop {
match swarm.next_event().await {
SwarmEvent::Behaviour(Event::InboundRequest { result, .. }) => {
assert!(result.is_ok());
break
},
_ => {},
}
}
}
}.boxed().into()
}).unwrap();
.boxed()
.into()
})
.unwrap();
// Remove and run the remaining swarm.
let (mut swarm, _) = swarms.remove(0);
@@ -1111,20 +1137,18 @@ mod tests {
);
assert!(response_receiver.is_none());
response_receiver = Some(receiver);
}
SwarmEvent::Behaviour(Event::RequestFinished {
result, ..
}) => {
},
SwarmEvent::Behaviour(Event::RequestFinished { result, .. }) => {
assert!(result.is_err());
break;
}
_ => {}
break
},
_ => {},
}
}
match response_receiver.unwrap().await.unwrap().unwrap_err() {
RequestFailure::Network(OutboundFailure::ConnectionClosed) => {},
_ => panic!()
_ => panic!(),
}
});
}
@@ -1197,89 +1221,97 @@ mod tests {
swarm_1.dial_addr(listen_add_2).unwrap();
// Run swarm 2 in the background, receiving two requests.
pool.spawner().spawn_obj(
async move {
loop {
match swarm_2.next_event().await {
SwarmEvent::Behaviour(Event::InboundRequest { result, .. }) => {
result.unwrap();
},
_ => {}
pool.spawner()
.spawn_obj(
async move {
loop {
match swarm_2.next_event().await {
SwarmEvent::Behaviour(Event::InboundRequest { result, .. }) => {
result.unwrap();
},
_ => {},
}
}
}
}.boxed().into()
).unwrap();
.boxed()
.into(),
)
.unwrap();
// Handle both requests sent by swarm 1 to swarm 2 in the background.
//
// Make sure both requests overlap, by answering the first only after receiving the
// second.
pool.spawner().spawn_obj(async move {
let protocol_1_request = swarm_2_handler_1.next().await;
let protocol_2_request = swarm_2_handler_2.next().await;
pool.spawner()
.spawn_obj(
async move {
let protocol_1_request = swarm_2_handler_1.next().await;
let protocol_2_request = swarm_2_handler_2.next().await;
protocol_1_request.unwrap()
.pending_response
.send(OutgoingResponse {
result: Ok(b"this is a response".to_vec()),
reputation_changes: Vec::new(),
sent_feedback: None,
})
.unwrap();
protocol_2_request.unwrap()
.pending_response
.send(OutgoingResponse {
result: Ok(b"this is a response".to_vec()),
reputation_changes: Vec::new(),
sent_feedback: None,
})
.unwrap();
}.boxed().into()).unwrap();
protocol_1_request
.unwrap()
.pending_response
.send(OutgoingResponse {
result: Ok(b"this is a response".to_vec()),
reputation_changes: Vec::new(),
sent_feedback: None,
})
.unwrap();
protocol_2_request
.unwrap()
.pending_response
.send(OutgoingResponse {
result: Ok(b"this is a response".to_vec()),
reputation_changes: Vec::new(),
sent_feedback: None,
})
.unwrap();
}
.boxed()
.into(),
)
.unwrap();
// Have swarm 1 send two requests to swarm 2 and await responses.
pool.run_until(
async move {
let mut response_receivers = None;
let mut num_responses = 0;
pool.run_until(async move {
let mut response_receivers = None;
let mut num_responses = 0;
loop {
match swarm_1.next_event().await {
SwarmEvent::ConnectionEstablished { peer_id, .. } => {
let (sender_1, receiver_1) = oneshot::channel();
let (sender_2, receiver_2) = oneshot::channel();
swarm_1.behaviour_mut().send_request(
&peer_id,
protocol_name_1,
b"this is a request".to_vec(),
sender_1,
IfDisconnected::ImmediateError,
);
swarm_1.behaviour_mut().send_request(
&peer_id,
protocol_name_2,
b"this is a request".to_vec(),
sender_2,
IfDisconnected::ImmediateError,
);
assert!(response_receivers.is_none());
response_receivers = Some((receiver_1, receiver_2));
loop {
match swarm_1.next_event().await {
SwarmEvent::ConnectionEstablished { peer_id, .. } => {
let (sender_1, receiver_1) = oneshot::channel();
let (sender_2, receiver_2) = oneshot::channel();
swarm_1.behaviour_mut().send_request(
&peer_id,
protocol_name_1,
b"this is a request".to_vec(),
sender_1,
IfDisconnected::ImmediateError,
);
swarm_1.behaviour_mut().send_request(
&peer_id,
protocol_name_2,
b"this is a request".to_vec(),
sender_2,
IfDisconnected::ImmediateError,
);
assert!(response_receivers.is_none());
response_receivers = Some((receiver_1, receiver_2));
},
SwarmEvent::Behaviour(Event::RequestFinished { result, .. }) => {
num_responses += 1;
result.unwrap();
if num_responses == 2 {
break
}
SwarmEvent::Behaviour(Event::RequestFinished {
result, ..
}) => {
num_responses += 1;
result.unwrap();
if num_responses == 2 {
break;
}
}
_ => {}
}
},
_ => {},
}
let (response_receiver_1, response_receiver_2) = response_receivers.unwrap();
assert_eq!(response_receiver_1.await.unwrap().unwrap(), b"this is a response");
assert_eq!(response_receiver_2.await.unwrap().unwrap(), b"this is a response");
}
);
let (response_receiver_1, response_receiver_2) = response_receivers.unwrap();
assert_eq!(response_receiver_1.await.unwrap().unwrap(), b"this is a response");
assert_eq!(response_receiver_2.await.unwrap().unwrap(), b"this is a response");
});
}
}
File diff suppressed because it is too large Load Diff
+27 -25
View File
@@ -18,10 +18,8 @@
use crate::transport::BandwidthSinks;
use prometheus_endpoint::{
self as prometheus,
Counter, CounterVec, Gauge, GaugeVec, HistogramOpts,
PrometheusError, Registry, U64, Opts,
SourcedCounter, SourcedGauge, MetricSource,
self as prometheus, Counter, CounterVec, Gauge, GaugeVec, HistogramOpts, MetricSource, Opts,
PrometheusError, Registry, SourcedCounter, SourcedGauge, U64,
};
use std::{
str,
@@ -267,13 +265,14 @@ impl BandwidthCounters {
/// Registers the `BandwidthCounters` metric whose values are
/// obtained from the given sinks.
fn register(registry: &Registry, sinks: Arc<BandwidthSinks>) -> Result<(), PrometheusError> {
prometheus::register(SourcedCounter::new(
&Opts::new(
"sub_libp2p_network_bytes_total",
"Total bandwidth usage"
).variable_label("direction"),
BandwidthCounters(sinks),
)?, registry)?;
prometheus::register(
SourcedCounter::new(
&Opts::new("sub_libp2p_network_bytes_total", "Total bandwidth usage")
.variable_label("direction"),
BandwidthCounters(sinks),
)?,
registry,
)?;
Ok(())
}
@@ -296,13 +295,16 @@ impl MajorSyncingGauge {
/// Registers the `MajorSyncGauge` metric whose value is
/// obtained from the given `AtomicBool`.
fn register(registry: &Registry, value: Arc<AtomicBool>) -> Result<(), PrometheusError> {
prometheus::register(SourcedGauge::new(
&Opts::new(
"sub_libp2p_is_major_syncing",
"Whether the node is performing a major sync or not.",
),
MajorSyncingGauge(value),
)?, registry)?;
prometheus::register(
SourcedGauge::new(
&Opts::new(
"sub_libp2p_is_major_syncing",
"Whether the node is performing a major sync or not.",
),
MajorSyncingGauge(value),
)?,
registry,
)?;
Ok(())
}
@@ -324,13 +326,13 @@ impl NumConnectedGauge {
/// Registers the `MajorSyncingGauge` metric whose value is
/// obtained from the given `AtomicUsize`.
fn register(registry: &Registry, value: Arc<AtomicUsize>) -> Result<(), PrometheusError> {
prometheus::register(SourcedGauge::new(
&Opts::new(
"sub_libp2p_peers_count",
"Number of connected peers",
),
NumConnectedGauge(value),
)?, registry)?;
prometheus::register(
SourcedGauge::new(
&Opts::new("sub_libp2p_peers_count", "Number of connected peers"),
NumConnectedGauge(value),
)?,
registry,
)?;
Ok(())
}
@@ -30,17 +30,18 @@
//! [`OutChannels::push`] to put the sender within a [`OutChannels`].
//! - Send events by calling [`OutChannels::send`]. Events are cloned for each sender in the
//! collection.
//!
use crate::Event;
use futures::{prelude::*, channel::mpsc, ready, stream::FusedStream};
use futures::{channel::mpsc, prelude::*, ready, stream::FusedStream};
use parking_lot::Mutex;
use prometheus_endpoint::{register, CounterVec, GaugeVec, Opts, PrometheusError, Registry, U64};
use std::{
convert::TryFrom as _,
fmt, pin::Pin, sync::Arc,
task::{Context, Poll}
fmt,
pin::Pin,
sync::Arc,
task::{Context, Poll},
};
/// Creates a new channel that can be associated to a [`OutChannels`].
@@ -100,8 +101,10 @@ impl Stream for Receiver {
let metrics = self.metrics.lock().clone();
match metrics.as_ref().map(|m| m.as_ref()) {
Some(Some(metrics)) => metrics.event_out(&ev, self.name),
Some(None) => (), // no registry
None => log::warn!("Inconsistency in out_events: event happened before sender associated"),
Some(None) => (), // no registry
None => log::warn!(
"Inconsistency in out_events: event happened before sender associated"
),
}
Poll::Ready(Some(ev))
} else {
@@ -136,16 +139,10 @@ pub struct OutChannels {
impl OutChannels {
/// Creates a new empty collection of senders.
pub fn new(registry: Option<&Registry>) -> Result<Self, PrometheusError> {
let metrics = if let Some(registry) = registry {
Some(Metrics::register(registry)?)
} else {
None
};
let metrics =
if let Some(registry) = registry { Some(Metrics::register(registry)?) } else { None };
Ok(OutChannels {
event_streams: Vec::new(),
metrics: Arc::new(metrics),
})
Ok(OutChannels { event_streams: Vec::new(), metrics: Arc::new(metrics) })
}
/// Adds a new [`Sender`] to the collection.
@@ -164,9 +161,8 @@ impl OutChannels {
/// Sends an event.
pub fn send(&mut self, event: Event) {
self.event_streams.retain(|sender| {
sender.inner.unbounded_send(event.clone()).is_ok()
});
self.event_streams
.retain(|sender| sender.inner.unbounded_send(event.clone()).is_ok());
if let Some(metrics) = &*self.metrics {
for ev in &self.event_streams {
@@ -223,20 +219,18 @@ impl Metrics {
fn event_in(&self, event: &Event, num: u64, name: &str) {
match event {
Event::Dht(_) => {
self.events_total
.with_label_values(&["dht", "sent", name])
.inc_by(num);
}
self.events_total.with_label_values(&["dht", "sent", name]).inc_by(num);
},
Event::SyncConnected { .. } => {
self.events_total
.with_label_values(&["sync-connected", "sent", name])
.inc_by(num);
}
},
Event::SyncDisconnected { .. } => {
self.events_total
.with_label_values(&["sync-disconnected", "sent", name])
.inc_by(num);
}
},
Event::NotificationStreamOpened { protocol, .. } => {
self.events_total
.with_label_values(&[&format!("notif-open-{:?}", protocol), "sent", name])
@@ -247,36 +241,31 @@ impl Metrics {
.with_label_values(&[&format!("notif-closed-{:?}", protocol), "sent", name])
.inc_by(num);
},
Event::NotificationsReceived { messages, .. } => {
Event::NotificationsReceived { messages, .. } =>
for (protocol, message) in messages {
self.events_total
.with_label_values(&[&format!("notif-{:?}", protocol), "sent", name])
.inc_by(num);
self.notifications_sizes
.with_label_values(&[protocol, "sent", name])
.inc_by(num.saturating_mul(u64::try_from(message.len()).unwrap_or(u64::MAX)));
}
},
self.notifications_sizes.with_label_values(&[protocol, "sent", name]).inc_by(
num.saturating_mul(u64::try_from(message.len()).unwrap_or(u64::MAX)),
);
},
}
}
fn event_out(&self, event: &Event, name: &str) {
match event {
Event::Dht(_) => {
self.events_total
.with_label_values(&["dht", "received", name])
.inc();
}
self.events_total.with_label_values(&["dht", "received", name]).inc();
},
Event::SyncConnected { .. } => {
self.events_total
.with_label_values(&["sync-connected", "received", name])
.inc();
}
self.events_total.with_label_values(&["sync-connected", "received", name]).inc();
},
Event::SyncDisconnected { .. } => {
self.events_total
.with_label_values(&["sync-disconnected", "received", name])
.inc();
}
},
Event::NotificationStreamOpened { protocol, .. } => {
self.events_total
.with_label_values(&[&format!("notif-open-{:?}", protocol), "received", name])
@@ -287,7 +276,7 @@ impl Metrics {
.with_label_values(&[&format!("notif-closed-{:?}", protocol), "received", name])
.inc();
},
Event::NotificationsReceived { messages, .. } => {
Event::NotificationsReceived { messages, .. } =>
for (protocol, message) in messages {
self.events_total
.with_label_values(&[&format!("notif-{:?}", protocol), "received", name])
@@ -295,8 +284,7 @@ impl Metrics {
self.notifications_sizes
.with_label_values(&[&protocol, "received", name])
.inc_by(u64::try_from(message.len()).unwrap_or(u64::MAX));
}
},
},
}
}
}
+151 -154
View File
@@ -16,13 +16,14 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{config, Event, NetworkService, NetworkWorker};
use crate::block_request_handler::BlockRequestHandler;
use crate::state_request_handler::StateRequestHandler;
use crate::light_client_requests::handler::LightClientRequestHandler;
use crate::{
block_request_handler::BlockRequestHandler, config,
light_client_requests::handler::LightClientRequestHandler,
state_request_handler::StateRequestHandler, Event, NetworkService, NetworkWorker,
};
use libp2p::PeerId;
use futures::prelude::*;
use libp2p::PeerId;
use sp_runtime::traits::{Block as BlockT, Header as _};
use std::{borrow::Cow, sync::Arc, time::Duration};
use substrate_test_runtime_client::{TestClientBuilder, TestClientBuilderExt as _};
@@ -37,14 +38,10 @@ type TestNetworkService = NetworkService<
///
/// > **Note**: We return the events stream in order to not possibly lose events between the
/// > construction of the service and the moment the events stream is grabbed.
fn build_test_full_node(config: config::NetworkConfiguration)
-> (Arc<TestNetworkService>, impl Stream<Item = Event>)
{
let client = Arc::new(
TestClientBuilder::with_default_backend()
.build_with_longest_chain()
.0,
);
fn build_test_full_node(
config: config::NetworkConfiguration,
) -> (Arc<TestNetworkService>, impl Stream<Item = Event>) {
let client = Arc::new(TestClientBuilder::with_default_backend().build_with_longest_chain().0);
#[derive(Clone)]
struct PassThroughVerifier(bool);
@@ -69,14 +66,13 @@ fn build_test_full_node(config: config::NetworkConfiguration)
.log(|l| {
l.try_as_raw(sp_runtime::generic::OpaqueDigestItemId::Consensus(b"aura"))
.or_else(|| {
l.try_as_raw(sp_runtime::generic::OpaqueDigestItemId::Consensus(b"babe"))
l.try_as_raw(sp_runtime::generic::OpaqueDigestItemId::Consensus(
b"babe",
))
})
})
.map(|blob| {
vec![(
sp_blockchain::well_known_cache_keys::AUTHORITIES,
blob.to_vec(),
)]
vec![(sp_blockchain::well_known_cache_keys::AUTHORITIES, blob.to_vec())]
});
let mut import = sp_consensus::BlockImportParams::new(origin, header);
@@ -99,30 +95,20 @@ fn build_test_full_node(config: config::NetworkConfiguration)
let protocol_id = config::ProtocolId::from("/test-protocol-name");
let block_request_protocol_config = {
let (handler, protocol_config) = BlockRequestHandler::new(
&protocol_id,
client.clone(),
50,
);
let (handler, protocol_config) = BlockRequestHandler::new(&protocol_id, client.clone(), 50);
async_std::task::spawn(handler.run().boxed());
protocol_config
};
let state_request_protocol_config = {
let (handler, protocol_config) = StateRequestHandler::new(
&protocol_id,
client.clone(),
50,
);
let (handler, protocol_config) = StateRequestHandler::new(&protocol_id, client.clone(), 50);
async_std::task::spawn(handler.run().boxed());
protocol_config
};
let light_client_request_protocol_config = {
let (handler, protocol_config) = LightClientRequestHandler::new(
&protocol_id,
client.clone(),
);
let (handler, protocol_config) =
LightClientRequestHandler::new(&protocol_id, client.clone());
async_std::task::spawn(handler.run().boxed());
protocol_config
};
@@ -130,7 +116,9 @@ fn build_test_full_node(config: config::NetworkConfiguration)
let worker = NetworkWorker::new(config::Params {
role: config::Role::Full,
executor: None,
transactions_handler_executor: Box::new(|task| { async_std::task::spawn(task); }),
transactions_handler_executor: Box::new(|task| {
async_std::task::spawn(task);
}),
network_config: config,
chain: client.clone(),
on_demand: None,
@@ -162,43 +150,42 @@ const PROTOCOL_NAME: Cow<'static, str> = Cow::Borrowed("/foo");
/// Builds two nodes and their associated events stream.
/// The nodes are connected together and have the `PROTOCOL_NAME` protocol registered.
fn build_nodes_one_proto()
-> (Arc<TestNetworkService>, impl Stream<Item = Event>, Arc<TestNetworkService>, impl Stream<Item = Event>)
{
fn build_nodes_one_proto() -> (
Arc<TestNetworkService>,
impl Stream<Item = Event>,
Arc<TestNetworkService>,
impl Stream<Item = Event>,
) {
let listen_addr = config::build_multiaddr![Memory(rand::random::<u64>())];
let (node1, events_stream1) = build_test_full_node(config::NetworkConfiguration {
extra_sets: vec![
config::NonDefaultSetConfig {
notifications_protocol: PROTOCOL_NAME,
fallback_names: Vec::new(),
max_notification_size: 1024 * 1024,
set_config: Default::default()
}
],
extra_sets: vec![config::NonDefaultSetConfig {
notifications_protocol: PROTOCOL_NAME,
fallback_names: Vec::new(),
max_notification_size: 1024 * 1024,
set_config: Default::default(),
}],
listen_addresses: vec![listen_addr.clone()],
transport: config::TransportConfig::MemoryOnly,
.. config::NetworkConfiguration::new_local()
..config::NetworkConfiguration::new_local()
});
let (node2, events_stream2) = build_test_full_node(config::NetworkConfiguration {
extra_sets: vec![
config::NonDefaultSetConfig {
notifications_protocol: PROTOCOL_NAME,
fallback_names: Vec::new(),
max_notification_size: 1024 * 1024,
set_config: config::SetConfig {
reserved_nodes: vec![config::MultiaddrWithPeerId {
multiaddr: listen_addr,
peer_id: node1.local_peer_id().clone(),
}],
.. Default::default()
}
}
],
extra_sets: vec![config::NonDefaultSetConfig {
notifications_protocol: PROTOCOL_NAME,
fallback_names: Vec::new(),
max_notification_size: 1024 * 1024,
set_config: config::SetConfig {
reserved_nodes: vec![config::MultiaddrWithPeerId {
multiaddr: listen_addr,
peer_id: node1.local_peer_id().clone(),
}],
..Default::default()
},
}],
listen_addresses: vec![],
transport: config::TransportConfig::MemoryOnly,
.. config::NetworkConfiguration::new_local()
..config::NetworkConfiguration::new_local()
});
(node1, events_stream1, node2, events_stream2)
@@ -214,10 +201,18 @@ fn notifications_state_consistent() {
// Write some initial notifications that shouldn't get through.
for _ in 0..(rand::random::<u8>() % 5) {
node1.write_notification(node2.local_peer_id().clone(), PROTOCOL_NAME, b"hello world".to_vec());
node1.write_notification(
node2.local_peer_id().clone(),
PROTOCOL_NAME,
b"hello world".to_vec(),
);
}
for _ in 0..(rand::random::<u8>() % 5) {
node2.write_notification(node1.local_peer_id().clone(), PROTOCOL_NAME, b"hello world".to_vec());
node2.write_notification(
node1.local_peer_id().clone(),
PROTOCOL_NAME,
b"hello world".to_vec(),
);
}
async_std::task::block_on(async move {
@@ -234,16 +229,24 @@ fn notifications_state_consistent() {
iterations += 1;
if iterations >= 1_000 {
assert!(something_happened);
break;
break
}
// Start by sending a notification from node1 to node2 and vice-versa. Part of the
// test consists in ensuring that notifications get ignored if the stream isn't open.
if rand::random::<u8>() % 5 >= 3 {
node1.write_notification(node2.local_peer_id().clone(), PROTOCOL_NAME, b"hello world".to_vec());
node1.write_notification(
node2.local_peer_id().clone(),
PROTOCOL_NAME,
b"hello world".to_vec(),
);
}
if rand::random::<u8>() % 5 >= 3 {
node2.write_notification(node1.local_peer_id().clone(), PROTOCOL_NAME, b"hello world".to_vec());
node2.write_notification(
node1.local_peer_id().clone(),
PROTOCOL_NAME,
b"hello world".to_vec(),
);
}
// Also randomly disconnect the two nodes from time to time.
@@ -272,32 +275,40 @@ fn notifications_state_consistent() {
};
match next_event {
future::Either::Left(Event::NotificationStreamOpened { remote, protocol, .. }) => {
future::Either::Left(Event::NotificationStreamOpened {
remote, protocol, ..
}) => {
something_happened = true;
assert!(!node1_to_node2_open);
node1_to_node2_open = true;
assert_eq!(remote, *node2.local_peer_id());
assert_eq!(protocol, PROTOCOL_NAME);
}
future::Either::Right(Event::NotificationStreamOpened { remote, protocol, .. }) => {
},
future::Either::Right(Event::NotificationStreamOpened {
remote, protocol, ..
}) => {
something_happened = true;
assert!(!node2_to_node1_open);
node2_to_node1_open = true;
assert_eq!(remote, *node1.local_peer_id());
assert_eq!(protocol, PROTOCOL_NAME);
}
future::Either::Left(Event::NotificationStreamClosed { remote, protocol, .. }) => {
},
future::Either::Left(Event::NotificationStreamClosed {
remote, protocol, ..
}) => {
assert!(node1_to_node2_open);
node1_to_node2_open = false;
assert_eq!(remote, *node2.local_peer_id());
assert_eq!(protocol, PROTOCOL_NAME);
}
future::Either::Right(Event::NotificationStreamClosed { remote, protocol, .. }) => {
},
future::Either::Right(Event::NotificationStreamClosed {
remote, protocol, ..
}) => {
assert!(node2_to_node1_open);
node2_to_node1_open = false;
assert_eq!(remote, *node1.local_peer_id());
assert_eq!(protocol, PROTOCOL_NAME);
}
},
future::Either::Left(Event::NotificationsReceived { remote, .. }) => {
assert!(node1_to_node2_open);
assert_eq!(remote, *node2.local_peer_id());
@@ -305,10 +316,10 @@ fn notifications_state_consistent() {
node1.write_notification(
node2.local_peer_id().clone(),
PROTOCOL_NAME,
b"hello world".to_vec()
b"hello world".to_vec(),
);
}
}
},
future::Either::Right(Event::NotificationsReceived { remote, .. }) => {
assert!(node2_to_node1_open);
assert_eq!(remote, *node1.local_peer_id());
@@ -316,18 +327,18 @@ fn notifications_state_consistent() {
node2.write_notification(
node1.local_peer_id().clone(),
PROTOCOL_NAME,
b"hello world".to_vec()
b"hello world".to_vec(),
);
}
}
},
// Add new events here.
future::Either::Left(Event::SyncConnected { .. }) => {}
future::Either::Right(Event::SyncConnected { .. }) => {}
future::Either::Left(Event::SyncDisconnected { .. }) => {}
future::Either::Right(Event::SyncDisconnected { .. }) => {}
future::Either::Left(Event::Dht(_)) => {}
future::Either::Right(Event::Dht(_)) => {}
future::Either::Left(Event::SyncConnected { .. }) => {},
future::Either::Right(Event::SyncConnected { .. }) => {},
future::Either::Left(Event::SyncDisconnected { .. }) => {},
future::Either::Right(Event::SyncDisconnected { .. }) => {},
future::Either::Left(Event::Dht(_)) => {},
future::Either::Right(Event::Dht(_)) => {},
};
}
});
@@ -339,19 +350,14 @@ fn lots_of_incoming_peers_works() {
let (main_node, _) = build_test_full_node(config::NetworkConfiguration {
listen_addresses: vec![listen_addr.clone()],
extra_sets: vec![
config::NonDefaultSetConfig {
notifications_protocol: PROTOCOL_NAME,
fallback_names: Vec::new(),
max_notification_size: 1024 * 1024,
set_config: config::SetConfig {
in_peers: u32::MAX,
.. Default::default()
},
}
],
extra_sets: vec![config::NonDefaultSetConfig {
notifications_protocol: PROTOCOL_NAME,
fallback_names: Vec::new(),
max_notification_size: 1024 * 1024,
set_config: config::SetConfig { in_peers: u32::MAX, ..Default::default() },
}],
transport: config::TransportConfig::MemoryOnly,
.. config::NetworkConfiguration::new_local()
..config::NetworkConfiguration::new_local()
});
let main_node_peer_id = main_node.local_peer_id().clone();
@@ -365,22 +371,20 @@ fn lots_of_incoming_peers_works() {
let (_dialing_node, event_stream) = build_test_full_node(config::NetworkConfiguration {
listen_addresses: vec![],
extra_sets: vec![
config::NonDefaultSetConfig {
notifications_protocol: PROTOCOL_NAME,
fallback_names: Vec::new(),
max_notification_size: 1024 * 1024,
set_config: config::SetConfig {
reserved_nodes: vec![config::MultiaddrWithPeerId {
multiaddr: listen_addr.clone(),
peer_id: main_node_peer_id.clone(),
}],
.. Default::default()
},
}
],
extra_sets: vec![config::NonDefaultSetConfig {
notifications_protocol: PROTOCOL_NAME,
fallback_names: Vec::new(),
max_notification_size: 1024 * 1024,
set_config: config::SetConfig {
reserved_nodes: vec![config::MultiaddrWithPeerId {
multiaddr: listen_addr.clone(),
peer_id: main_node_peer_id.clone(),
}],
..Default::default()
},
}],
transport: config::TransportConfig::MemoryOnly,
.. config::NetworkConfiguration::new_local()
..config::NetworkConfiguration::new_local()
});
background_tasks_to_wait.push(async_std::task::spawn(async move {
@@ -416,9 +420,7 @@ fn lots_of_incoming_peers_works() {
}));
}
futures::executor::block_on(async move {
future::join_all(background_tasks_to_wait).await
});
futures::executor::block_on(async move { future::join_all(background_tasks_to_wait).await });
}
#[test]
@@ -437,14 +439,13 @@ fn notifications_back_pressure() {
while received_notifications < TOTAL_NOTIFS {
match events_stream2.next().await.unwrap() {
Event::NotificationStreamClosed { .. } => panic!(),
Event::NotificationsReceived { messages, .. } => {
Event::NotificationsReceived { messages, .. } =>
for message in messages {
assert_eq!(message.0, PROTOCOL_NAME);
assert_eq!(message.1, format!("hello #{}", received_notifications));
received_notifications += 1;
}
}
_ => {}
},
_ => {},
};
if rand::random::<u8>() < 2 {
@@ -458,7 +459,7 @@ fn notifications_back_pressure() {
loop {
match events_stream1.next().await.unwrap() {
Event::NotificationStreamOpened { .. } => break,
_ => {}
_ => {},
};
}
@@ -483,37 +484,33 @@ fn fallback_name_working() {
let listen_addr = config::build_multiaddr![Memory(rand::random::<u64>())];
let (node1, mut events_stream1) = build_test_full_node(config::NetworkConfiguration {
extra_sets: vec![
config::NonDefaultSetConfig {
notifications_protocol: NEW_PROTOCOL_NAME.clone(),
fallback_names: vec![PROTOCOL_NAME],
max_notification_size: 1024 * 1024,
set_config: Default::default()
}
],
extra_sets: vec![config::NonDefaultSetConfig {
notifications_protocol: NEW_PROTOCOL_NAME.clone(),
fallback_names: vec![PROTOCOL_NAME],
max_notification_size: 1024 * 1024,
set_config: Default::default(),
}],
listen_addresses: vec![listen_addr.clone()],
transport: config::TransportConfig::MemoryOnly,
.. config::NetworkConfiguration::new_local()
..config::NetworkConfiguration::new_local()
});
let (_, mut events_stream2) = build_test_full_node(config::NetworkConfiguration {
extra_sets: vec![
config::NonDefaultSetConfig {
notifications_protocol: PROTOCOL_NAME,
fallback_names: Vec::new(),
max_notification_size: 1024 * 1024,
set_config: config::SetConfig {
reserved_nodes: vec![config::MultiaddrWithPeerId {
multiaddr: listen_addr,
peer_id: node1.local_peer_id().clone(),
}],
.. Default::default()
}
}
],
extra_sets: vec![config::NonDefaultSetConfig {
notifications_protocol: PROTOCOL_NAME,
fallback_names: Vec::new(),
max_notification_size: 1024 * 1024,
set_config: config::SetConfig {
reserved_nodes: vec![config::MultiaddrWithPeerId {
multiaddr: listen_addr,
peer_id: node1.local_peer_id().clone(),
}],
..Default::default()
},
}],
listen_addresses: vec![],
transport: config::TransportConfig::MemoryOnly,
.. config::NetworkConfiguration::new_local()
..config::NetworkConfiguration::new_local()
});
let receiver = async_std::task::spawn(async move {
@@ -525,7 +522,7 @@ fn fallback_name_working() {
assert_eq!(negotiated_fallback, None);
break
},
_ => {}
_ => {},
};
}
});
@@ -539,7 +536,7 @@ fn fallback_name_working() {
assert_eq!(negotiated_fallback, Some(PROTOCOL_NAME));
break
},
_ => {}
_ => {},
};
}
@@ -555,7 +552,7 @@ fn ensure_listen_addresses_consistent_with_transport_memory() {
let _ = build_test_full_node(config::NetworkConfiguration {
listen_addresses: vec![listen_addr.clone()],
transport: config::TransportConfig::MemoryOnly,
.. config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None)
..config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None)
});
}
@@ -566,7 +563,7 @@ fn ensure_listen_addresses_consistent_with_transport_not_memory() {
let _ = build_test_full_node(config::NetworkConfiguration {
listen_addresses: vec![listen_addr.clone()],
.. config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None)
..config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None)
});
}
@@ -583,7 +580,7 @@ fn ensure_boot_node_addresses_consistent_with_transport_memory() {
listen_addresses: vec![listen_addr.clone()],
transport: config::TransportConfig::MemoryOnly,
boot_nodes: vec![boot_node],
.. config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None)
..config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None)
});
}
@@ -599,7 +596,7 @@ fn ensure_boot_node_addresses_consistent_with_transport_not_memory() {
let _ = build_test_full_node(config::NetworkConfiguration {
listen_addresses: vec![listen_addr.clone()],
boot_nodes: vec![boot_node],
.. config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None)
..config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None)
});
}
@@ -617,9 +614,9 @@ fn ensure_reserved_node_addresses_consistent_with_transport_memory() {
transport: config::TransportConfig::MemoryOnly,
default_peers_set: config::SetConfig {
reserved_nodes: vec![reserved_node],
.. Default::default()
..Default::default()
},
.. config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None)
..config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None)
});
}
@@ -636,9 +633,9 @@ fn ensure_reserved_node_addresses_consistent_with_transport_not_memory() {
listen_addresses: vec![listen_addr.clone()],
default_peers_set: config::SetConfig {
reserved_nodes: vec![reserved_node],
.. Default::default()
..Default::default()
},
.. config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None)
..config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None)
});
}
@@ -652,7 +649,7 @@ fn ensure_public_addresses_consistent_with_transport_memory() {
listen_addresses: vec![listen_addr.clone()],
transport: config::TransportConfig::MemoryOnly,
public_addresses: vec![public_address],
.. config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None)
..config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None)
});
}
@@ -665,6 +662,6 @@ fn ensure_public_addresses_consistent_with_transport_not_memory() {
let _ = build_test_full_node(config::NetworkConfiguration {
listen_addresses: vec![listen_addr.clone()],
public_addresses: vec![public_address],
.. config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None)
..config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None)
});
}
@@ -17,22 +17,27 @@
//! Helper for handling (i.e. answering) state requests from a remote peer via the
//! [`crate::request_responses::RequestResponsesBehaviour`].
use codec::{Encode, Decode};
use crate::chain::Client;
use crate::config::ProtocolId;
use crate::request_responses::{IncomingRequest, OutgoingResponse, ProtocolConfig};
use crate::schema::v1::{StateResponse, StateRequest, StateEntry};
use crate::{PeerId, ReputationChange};
use futures::channel::{mpsc, oneshot};
use futures::stream::StreamExt;
use crate::{
chain::Client,
config::ProtocolId,
request_responses::{IncomingRequest, OutgoingResponse, ProtocolConfig},
schema::v1::{StateEntry, StateRequest, StateResponse},
PeerId, ReputationChange,
};
use codec::{Decode, Encode};
use futures::{
channel::{mpsc, oneshot},
stream::StreamExt,
};
use log::debug;
use lru::LruCache;
use prost::Message;
use sp_runtime::generic::BlockId;
use sp_runtime::traits::Block as BlockT;
use std::sync::Arc;
use std::time::Duration;
use std::hash::{Hasher, Hash};
use sp_runtime::{generic::BlockId, traits::Block as BlockT};
use std::{
hash::{Hash, Hasher},
sync::Arc,
time::Duration,
};
const LOG_TARGET: &str = "sync";
const MAX_RESPONSE_BYTES: usize = 2 * 1024 * 1024; // Actual reponse may be bigger.
@@ -127,9 +132,7 @@ impl<B: BlockT> StateRequestHandler<B> {
Ok(()) => debug!(target: LOG_TARGET, "Handled block request from {}.", peer),
Err(e) => debug!(
target: LOG_TARGET,
"Failed to handle state request from {}: {}",
peer,
e,
"Failed to handle state request from {}: {}", peer, e,
),
}
}
@@ -144,11 +147,8 @@ impl<B: BlockT> StateRequestHandler<B> {
let request = StateRequest::decode(&payload[..])?;
let block: B::Hash = Decode::decode(&mut request.block.as_ref())?;
let key = SeenRequestsKey {
peer: *peer,
block: block.clone(),
start: request.start.clone(),
};
let key =
SeenRequestsKey { peer: *peer, block: block.clone(), start: request.start.clone() };
let mut reputation_changes = Vec::new();
@@ -163,7 +163,7 @@ impl<B: BlockT> StateRequestHandler<B> {
},
None => {
self.seen_requests.put(key.clone(), SeenRequestsValue::First);
}
},
}
log::trace!(
@@ -194,7 +194,8 @@ impl<B: BlockT> StateRequestHandler<B> {
&request.start,
MAX_RESPONSE_BYTES,
)?;
response.entries = entries.into_iter().map(|(key, value)| StateEntry { key, value }).collect();
response.entries =
entries.into_iter().map(|(key, value)| StateEntry { key, value }).collect();
if response.entries.is_empty() {
response.complete = true;
}
@@ -224,11 +225,9 @@ impl<B: BlockT> StateRequestHandler<B> {
Err(())
};
pending_response.send(OutgoingResponse {
result,
reputation_changes,
sent_feedback: None,
}).map_err(|_| HandleRequestError::SendResponse)
pending_response
.send(OutgoingResponse { result, reputation_changes, sent_feedback: None })
.map_err(|_| HandleRequestError::SendResponse)
}
}
+65 -57
View File
@@ -25,26 +25,35 @@
//! configuration as an extra peers set.
//! - Use [`TransactionsHandlerPrototype::build`] then [`TransactionsHandler::run`] to obtain a
//! `Future` that processes transactions.
//!
use crate::{
ExHashT, Event, ObservedRole,
config::{self, ProtocolId, TransactionPool, TransactionImportFuture, TransactionImport},
error, protocol::message, service::NetworkService, utils::{interval, LruHashSet},
config::{self, ProtocolId, TransactionImport, TransactionImportFuture, TransactionPool},
error,
protocol::message,
service::NetworkService,
utils::{interval, LruHashSet},
Event, ExHashT, ObservedRole,
};
use codec::{Decode, Encode};
use futures::{channel::mpsc, prelude::*, stream::FuturesUnordered};
use libp2p::{multiaddr, PeerId};
use log::{trace, debug, warn};
use prometheus_endpoint::{
Registry, Counter, PrometheusError, register, U64
};
use log::{debug, trace, warn};
use prometheus_endpoint::{register, Counter, PrometheusError, Registry, U64};
use sp_runtime::traits::Block as BlockT;
use std::borrow::Cow;
use std::collections::{HashMap, hash_map::Entry};
use std::sync::{atomic::{AtomicBool, Ordering}, Arc};
use std::{iter, num::NonZeroUsize, pin::Pin, task::Poll, time};
use std::{
borrow::Cow,
collections::{hash_map::Entry, HashMap},
iter,
num::NonZeroUsize,
pin::Pin,
sync::{
atomic::{AtomicBool, Ordering},
Arc,
},
task::Poll,
time,
};
/// Interval at which we propagate transactions;
const PROPAGATE_TIMEOUT: time::Duration = time::Duration::from_millis(2900);
@@ -84,10 +93,13 @@ struct Metrics {
impl Metrics {
fn register(r: &Registry) -> Result<Self, PrometheusError> {
Ok(Metrics {
propagated_transactions: register(Counter::new(
"sync_propagated_transactions",
"Number of transactions propagated to at least one peer",
)?, r)?,
propagated_transactions: register(
Counter::new(
"sync_propagated_transactions",
"Number of transactions propagated to at least one peer",
)?,
r,
)?,
})
}
}
@@ -106,7 +118,7 @@ impl<H: ExHashT> Future for PendingTransaction<H> {
let mut this = self.project();
if let Poll::Ready(import_result) = Pin::new(&mut this.validation).poll_unpin(cx) {
return Poll::Ready((this.tx_hash.clone(), import_result));
return Poll::Ready((this.tx_hash.clone(), import_result))
}
Poll::Pending
@@ -128,7 +140,7 @@ impl TransactionsHandlerPrototype {
proto.push_str(protocol_id.as_ref());
proto.push_str("/transactions/1");
proto
})
}),
}
}
@@ -143,7 +155,7 @@ impl TransactionsHandlerPrototype {
out_peers: 0,
reserved_nodes: Vec::new(),
non_reserved_mode: config::NonReservedPeerMode::Deny,
}
},
}
}
@@ -182,10 +194,7 @@ impl TransactionsHandlerPrototype {
},
};
let controller = TransactionsHandlerController {
to_handler,
gossip_enabled,
};
let controller = TransactionsHandlerController { to_handler, gossip_enabled };
Ok((handler, controller))
}
@@ -264,7 +273,7 @@ impl<B: BlockT + 'static, H: ExHashT> TransactionsHandler<B, H> {
/// interrupted.
pub async fn run(mut self) {
loop {
futures::select!{
futures::select! {
_ = self.propagate_timeout.next().fuse() => {
self.propagate_transactions();
},
@@ -301,7 +310,7 @@ impl<B: BlockT + 'static, H: ExHashT> TransactionsHandler<B, H> {
.collect::<multiaddr::Multiaddr>();
let result = self.service.add_peers_to_reserved_set(
self.protocol_name.clone(),
iter::once(addr).collect()
iter::once(addr).collect(),
);
if let Err(err) = result {
log::error!(target: "sync", "Add reserved peer failed: {}", err);
@@ -312,22 +321,30 @@ impl<B: BlockT + 'static, H: ExHashT> TransactionsHandler<B, H> {
.collect::<multiaddr::Multiaddr>();
let result = self.service.remove_peers_from_reserved_set(
self.protocol_name.clone(),
iter::once(addr).collect()
iter::once(addr).collect(),
);
if let Err(err) = result {
log::error!(target: "sync", "Removing reserved peer failed: {}", err);
}
},
Event::NotificationStreamOpened { remote, protocol, role, .. } if protocol == self.protocol_name => {
let _was_in = self.peers.insert(remote, Peer {
known_transactions: LruHashSet::new(NonZeroUsize::new(MAX_KNOWN_TRANSACTIONS)
.expect("Constant is nonzero")),
role,
});
Event::NotificationStreamOpened { remote, protocol, role, .. }
if protocol == self.protocol_name =>
{
let _was_in = self.peers.insert(
remote,
Peer {
known_transactions: LruHashSet::new(
NonZeroUsize::new(MAX_KNOWN_TRANSACTIONS).expect("Constant is nonzero"),
),
role,
},
);
debug_assert!(_was_in.is_none());
}
Event::NotificationStreamClosed { remote, protocol } if protocol == self.protocol_name => {
Event::NotificationStreamClosed { remote, protocol }
if protocol == self.protocol_name =>
{
let _peer = self.peers.remove(&remote);
debug_assert!(_peer.is_some());
}
@@ -335,7 +352,7 @@ impl<B: BlockT + 'static, H: ExHashT> TransactionsHandler<B, H> {
Event::NotificationsReceived { remote, messages } => {
for (protocol, message) in messages {
if protocol != self.protocol_name {
continue;
continue
}
if let Ok(m) = <message::Transactions<B::Extrinsic> as Decode>::decode(
@@ -349,28 +366,24 @@ impl<B: BlockT + 'static, H: ExHashT> TransactionsHandler<B, H> {
},
// Not our concern.
Event::NotificationStreamOpened { .. } | Event::NotificationStreamClosed { .. } => {}
Event::NotificationStreamOpened { .. } | Event::NotificationStreamClosed { .. } => {},
}
}
/// Called when peer sends us new transactions
fn on_transactions(
&mut self,
who: PeerId,
transactions: message::Transactions<B::Extrinsic>,
) {
fn on_transactions(&mut self, who: PeerId, transactions: message::Transactions<B::Extrinsic>) {
// sending transaction to light node is considered a bad behavior
if matches!(self.local_role, config::Role::Light) {
debug!(target: "sync", "Peer {} is trying to send transactions to the light node", who);
self.service.disconnect_peer(who, self.protocol_name.clone());
self.service.report_peer(who, rep::UNEXPECTED_TRANSACTIONS);
return;
return
}
// Accept transactions only when enabled
if !self.gossip_enabled.load(Ordering::Relaxed) {
trace!(target: "sync", "{} Ignoring transactions while disabled", who);
return;
return
}
trace!(target: "sync", "Received {} transactions from {}", transactions.len(), who);
@@ -382,7 +395,7 @@ impl<B: BlockT + 'static, H: ExHashT> TransactionsHandler<B, H> {
"Ignoring any further transactions that exceed `MAX_PENDING_TRANSACTIONS`({}) limit",
MAX_PENDING_TRANSACTIONS,
);
break;
break
}
let hash = self.transaction_pool.hash_of(&t);
@@ -400,7 +413,7 @@ impl<B: BlockT + 'static, H: ExHashT> TransactionsHandler<B, H> {
},
Entry::Occupied(mut entry) => {
entry.get_mut().push(who.clone());
}
},
}
}
}
@@ -408,7 +421,8 @@ impl<B: BlockT + 'static, H: ExHashT> TransactionsHandler<B, H> {
fn on_handle_transaction_import(&mut self, who: PeerId, import: TransactionImport) {
match import {
TransactionImport::KnownGood => self.service.report_peer(who, rep::ANY_TRANSACTION_REFUND),
TransactionImport::KnownGood =>
self.service.report_peer(who, rep::ANY_TRANSACTION_REFUND),
TransactionImport::NewGood => self.service.report_peer(who, rep::GOOD_TRANSACTION),
TransactionImport::Bad => self.service.report_peer(who, rep::BAD_TRANSACTION),
TransactionImport::None => {},
@@ -416,14 +430,11 @@ impl<B: BlockT + 'static, H: ExHashT> TransactionsHandler<B, H> {
}
/// Propagate one transaction.
pub fn propagate_transaction(
&mut self,
hash: &H,
) {
pub fn propagate_transaction(&mut self, hash: &H) {
debug!(target: "sync", "Propagating transaction [{:?}]", hash);
// Accept transactions only when enabled
if !self.gossip_enabled.load(Ordering::Relaxed) {
return;
return
}
if let Some(transaction) = self.transaction_pool.transaction(hash) {
let propagated_to = self.do_propagate_transactions(&[(hash.clone(), transaction)]);
@@ -441,7 +452,7 @@ impl<B: BlockT + 'static, H: ExHashT> TransactionsHandler<B, H> {
for (who, peer) in self.peers.iter_mut() {
// never send transactions to the light node
if matches!(peer.role, ObservedRole::Light) {
continue;
continue
}
let (hashes, to_send): (Vec<_>, Vec<_>) = transactions
@@ -454,16 +465,13 @@ impl<B: BlockT + 'static, H: ExHashT> TransactionsHandler<B, H> {
if !to_send.is_empty() {
for hash in hashes {
propagated_to
.entry(hash)
.or_default()
.push(who.to_base58());
propagated_to.entry(hash).or_default().push(who.to_base58());
}
trace!(target: "sync", "Sending {} transactions to {}", to_send.len(), who);
self.service.write_notification(
who.clone(),
self.protocol_name.clone(),
to_send.encode()
to_send.encode(),
);
}
}
@@ -479,7 +487,7 @@ impl<B: BlockT + 'static, H: ExHashT> TransactionsHandler<B, H> {
fn propagate_transactions(&mut self) {
// Accept transactions only when enabled
if !self.gossip_enabled.load(Ordering::Relaxed) {
return;
return
}
debug!(target: "sync", "Propagating transactions");
let transactions = self.transaction_pool.transactions();
+25 -20
View File
@@ -17,15 +17,18 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use libp2p::{
PeerId, Transport,
bandwidth,
core::{
self, either::EitherTransport, muxing::StreamMuxerBox,
transport::{Boxed, OptionalTransport}, upgrade
self,
either::EitherTransport,
muxing::StreamMuxerBox,
transport::{Boxed, OptionalTransport},
upgrade,
},
mplex, identity, bandwidth, wasm_ext, noise
identity, mplex, noise, wasm_ext, PeerId, Transport,
};
#[cfg(not(target_os = "unknown"))]
use libp2p::{tcp, dns, websocket};
use libp2p::{dns, tcp, websocket};
use std::{sync::Arc, time::Duration};
pub use self::bandwidth::BandwidthSinks;
@@ -61,8 +64,8 @@ pub fn build_transport(
#[cfg(not(target_os = "unknown"))]
let transport = transport.or_transport(if !memory_only {
let desktop_trans = tcp::TcpConfig::new().nodelay(true);
let desktop_trans = websocket::WsConfig::new(desktop_trans.clone())
.or_transport(desktop_trans);
let desktop_trans =
websocket::WsConfig::new(desktop_trans.clone()).or_transport(desktop_trans);
let dns_init = futures::executor::block_on(dns::DnsConfig::system(desktop_trans.clone()));
OptionalTransport::some(if let Ok(dns) = dns_init {
EitherTransport::Left(dns)
@@ -81,23 +84,24 @@ pub fn build_transport(
let (transport, bandwidth) = bandwidth::BandwidthLogging::new(transport);
let authentication_config = {
// For more information about these two panics, see in "On the Importance of
// Checking Cryptographic Protocols for Faults" by Dan Boneh, Richard A. DeMillo,
// and Richard J. Lipton.
let noise_keypair = noise::Keypair::<noise::X25519Spec>::new().into_authentic(&keypair)
let authentication_config =
{
// For more information about these two panics, see in "On the Importance of
// Checking Cryptographic Protocols for Faults" by Dan Boneh, Richard A. DeMillo,
// and Richard J. Lipton.
let noise_keypair = noise::Keypair::<noise::X25519Spec>::new().into_authentic(&keypair)
.expect("can only fail in case of a hardware bug; since this signing is performed only \
once and at initialization, we're taking the bet that the inconvenience of a very \
rare panic here is basically zero");
// Legacy noise configurations for backward compatibility.
let mut noise_legacy = noise::LegacyConfig::default();
noise_legacy.recv_legacy_handshake = true;
// Legacy noise configurations for backward compatibility.
let mut noise_legacy = noise::LegacyConfig::default();
noise_legacy.recv_legacy_handshake = true;
let mut xx_config = noise::NoiseConfig::xx(noise_keypair);
xx_config.set_legacy_config(noise_legacy.clone());
xx_config.into_authenticated()
};
let mut xx_config = noise::NoiseConfig::xx(noise_keypair);
xx_config.set_legacy_config(noise_legacy.clone());
xx_config.into_authenticated()
};
let multiplexing_config = {
let mut mplex_config = mplex::MplexConfig::new();
@@ -117,7 +121,8 @@ pub fn build_transport(
core::upgrade::SelectUpgrade::new(yamux_config, mplex_config)
};
let transport = transport.upgrade(upgrade::Version::V1Lazy)
let transport = transport
.upgrade(upgrade::Version::V1Lazy)
.authenticate(authentication_config)
.multiplex(multiplexing_config)
.timeout(Duration::from_secs(20))
+3 -7
View File
@@ -19,8 +19,7 @@
use futures::{stream::unfold, FutureExt, Stream, StreamExt};
use futures_timer::Delay;
use linked_hash_set::LinkedHashSet;
use std::time::Duration;
use std::{hash::Hash, num::NonZeroUsize};
use std::{hash::Hash, num::NonZeroUsize, time::Duration};
/// Creates a stream that returns a new value every `duration`.
pub fn interval(duration: Duration) -> impl Stream<Item = ()> + Unpin {
@@ -39,10 +38,7 @@ pub struct LruHashSet<T: Hash + Eq> {
impl<T: Hash + Eq> LruHashSet<T> {
/// Create a new `LruHashSet` with the given (exclusive) limit.
pub fn new(limit: NonZeroUsize) -> Self {
Self {
set: LinkedHashSet::new(),
limit,
}
Self { set: LinkedHashSet::new(), limit }
}
/// Insert element into the set.
@@ -55,7 +51,7 @@ impl<T: Hash + Eq> LruHashSet<T> {
if self.set.len() == usize::from(self.limit) {
self.set.pop_front(); // remove oldest entry
}
return true;
return true
}
false
}