// Copyright 2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see .
//! The Network Bridge Subsystem - protocol multiplexer for Polkadot.
#![deny(unused_crate_dependencies)]
#![warn(missing_docs)]
use always_assert::never;
use bytes::Bytes;
use futures::{prelude::*, stream::BoxStream};
use parity_scale_codec::{Decode, DecodeAll, Encode};
use parking_lot::Mutex;
use sc_network::Event as NetworkEvent;
use sp_consensus::SyncOracle;
use polkadot_node_network_protocol::{
self as net_protocol,
peer_set::{PeerSet, PerPeerSet},
v1 as protocol_v1, ObservedRole, OurView, PeerId, ProtocolVersion,
UnifiedReputationChange as Rep, Versioned, View,
};
use polkadot_node_subsystem::{
errors::{SubsystemError, SubsystemResult},
messages::{
network_bridge_event::{NewGossipTopology, TopologyPeerInfo},
ApprovalDistributionMessage, BitfieldDistributionMessage, CollatorProtocolMessage,
GossipSupportMessage, NetworkBridgeEvent, NetworkBridgeMessage,
StatementDistributionMessage,
},
overseer, ActivatedLeaf, ActiveLeavesUpdate, FromOrchestra, OverseerSignal, SpawnedSubsystem,
};
use polkadot_overseer::gen::OrchestraError as OverseerError;
use polkadot_primitives::v2::{AuthorityDiscoveryId, BlockNumber, Hash, ValidatorIndex};
/// Peer set info for network initialization.
///
/// To be added to [`NetworkConfiguration::extra_sets`].
pub use polkadot_node_network_protocol::peer_set::{peer_sets_info, IsAuthority};
use std::{
collections::{hash_map, HashMap},
iter::ExactSizeIterator,
sync::Arc,
};
mod validator_discovery;
/// Actual interfacing to the network based on the `Network` trait.
///
/// Defines the `Network` trait with an implementation for an `Arc`.
mod network;
use network::{send_message, Network};
use crate::network::get_peer_id_by_authority_id;
mod metrics;
use self::metrics::Metrics;
#[cfg(test)]
mod tests;
/// The maximum amount of heads a peer is allowed to have in their view at any time.
///
/// We use the same limit to compute the view sent to peers locally.
const MAX_VIEW_HEADS: usize = 5;
const MALFORMED_MESSAGE_COST: Rep = Rep::CostMajor("Malformed Network-bridge message");
const UNCONNECTED_PEERSET_COST: Rep = Rep::CostMinor("Message sent to un-connected peer-set");
const MALFORMED_VIEW_COST: Rep = Rep::CostMajor("Malformed view");
const EMPTY_VIEW_COST: Rep = Rep::CostMajor("Peer sent us an empty view");
// network bridge log target
const LOG_TARGET: &'static str = "parachain::network-bridge";
/// Messages from and to the network.
///
/// As transmitted to and received from subsystems.
#[derive(Debug, Encode, Decode, Clone)]
pub enum WireMessage {
/// A message from a peer on a specific protocol.
#[codec(index = 1)]
ProtocolMessage(M),
/// A view update from a peer.
#[codec(index = 2)]
ViewUpdate(View),
}
/// The network bridge subsystem.
pub struct NetworkBridge {
/// `Network` trait implementing type.
network_service: N,
authority_discovery_service: AD,
sync_oracle: Box,
metrics: Metrics,
}
impl NetworkBridge {
/// Create a new network bridge subsystem with underlying network service and authority discovery service.
///
/// This assumes that the network service has had the notifications protocol for the network
/// bridge already registered. See [`peers_sets_info`](peers_sets_info).
pub fn new(
network_service: N,
authority_discovery_service: AD,
sync_oracle: Box,
metrics: Metrics,
) -> Self {
NetworkBridge { network_service, authority_discovery_service, sync_oracle, metrics }
}
}
#[overseer::subsystem(NetworkBridge, error = SubsystemError, prefix = self::overseer)]
impl NetworkBridge
where
Net: Network + Sync,
AD: validator_discovery::AuthorityDiscovery + Clone,
{
fn start(mut self, ctx: Context) -> SpawnedSubsystem {
// The stream of networking events has to be created at initialization, otherwise the
// networking might open connections before the stream of events has been grabbed.
let network_stream = self.network_service.event_stream();
// Swallow error because failure is fatal to the node and we log with more precision
// within `run_network`.
let future = run_network(self, ctx, network_stream)
.map_err(|e| SubsystemError::with_origin("network-bridge", e))
.boxed();
SpawnedSubsystem { name: "network-bridge-subsystem", future }
}
}
struct PeerData {
/// The Latest view sent by the peer.
view: View,
version: ProtocolVersion,
}
#[derive(Debug)]
enum UnexpectedAbort {
/// Received error from overseer:
SubsystemError(SubsystemError),
/// The stream of incoming events concluded.
EventStreamConcluded,
}
impl From for UnexpectedAbort {
fn from(e: SubsystemError) -> Self {
UnexpectedAbort::SubsystemError(e)
}
}
impl From for UnexpectedAbort {
fn from(e: OverseerError) -> Self {
UnexpectedAbort::SubsystemError(SubsystemError::from(e))
}
}
#[derive(Default, Clone)]
struct Shared(Arc>);
#[derive(Default)]
struct SharedInner {
local_view: Option,
validation_peers: HashMap,
collation_peers: HashMap,
}
enum Mode {
Syncing(Box),
Active,
}
#[overseer::contextbounds(NetworkBridge, prefix = self::overseer)]
async fn handle_subsystem_messages(
mut ctx: Context,
mut network_service: N,
mut authority_discovery_service: AD,
shared: Shared,
sync_oracle: Box,
metrics: Metrics,
) -> Result<(), UnexpectedAbort>
where
N: Network,
AD: validator_discovery::AuthorityDiscovery + Clone,
{
// This is kept sorted, descending, by block number.
let mut live_heads: Vec = Vec::with_capacity(MAX_VIEW_HEADS);
let mut finalized_number = 0;
let mut validator_discovery = validator_discovery::Service::::new();
let mut mode = Mode::Syncing(sync_oracle);
loop {
futures::select! {
msg = ctx.recv().fuse() => match msg {
Ok(FromOrchestra::Signal(OverseerSignal::ActiveLeaves(active_leaves))) => {
let ActiveLeavesUpdate { activated, deactivated } = active_leaves;
gum::trace!(
target: LOG_TARGET,
action = "ActiveLeaves",
has_activated = activated.is_some(),
num_deactivated = %deactivated.len(),
);
for activated in activated {
let pos = live_heads
.binary_search_by(|probe| probe.number.cmp(&activated.number).reverse())
.unwrap_or_else(|i| i);
live_heads.insert(pos, activated);
}
live_heads.retain(|h| !deactivated.contains(&h.hash));
// if we're done syncing, set the mode to `Mode::Active`.
// Otherwise, we don't need to send view updates.
{
let is_done_syncing = match mode {
Mode::Active => true,
Mode::Syncing(ref mut sync_oracle) => !sync_oracle.is_major_syncing(),
};
if is_done_syncing {
mode = Mode::Active;
update_our_view(
&mut network_service,
&mut ctx,
&live_heads,
&shared,
finalized_number,
&metrics,
);
}
}
}
Ok(FromOrchestra::Signal(OverseerSignal::BlockFinalized(_hash, number))) => {
gum::trace!(
target: LOG_TARGET,
action = "BlockFinalized"
);
debug_assert!(finalized_number < number);
// we don't send the view updates here, but delay them until the next `ActiveLeaves`
// otherwise it might break assumptions of some of the subsystems
// that we never send the same `ActiveLeavesUpdate`
finalized_number = number;
}
Ok(FromOrchestra::Signal(OverseerSignal::Conclude)) => {
return Ok(());
}
Ok(FromOrchestra::Communication { msg }) => match msg {
NetworkBridgeMessage::ReportPeer(peer, rep) => {
if !rep.is_benefit() {
gum::debug!(
target: LOG_TARGET,
?peer,
?rep,
action = "ReportPeer"
);
}
metrics.on_report_event();
network_service.report_peer(peer, rep);
}
NetworkBridgeMessage::DisconnectPeer(peer, peer_set) => {
gum::trace!(
target: LOG_TARGET,
action = "DisconnectPeer",
?peer,
peer_set = ?peer_set,
);
network_service.disconnect_peer(peer, peer_set);
}
NetworkBridgeMessage::SendValidationMessage(peers, msg) => {
gum::trace!(
target: LOG_TARGET,
action = "SendValidationMessages",
num_messages = 1usize,
);
match msg {
Versioned::V1(msg) => send_validation_message_v1(
&mut network_service,
peers,
WireMessage::ProtocolMessage(msg),
&metrics,
),
}
}
NetworkBridgeMessage::SendValidationMessages(msgs) => {
gum::trace!(
target: LOG_TARGET,
action = "SendValidationMessages",
num_messages = %msgs.len(),
);
for (peers, msg) in msgs {
match msg {
Versioned::V1(msg) => send_validation_message_v1(
&mut network_service,
peers,
WireMessage::ProtocolMessage(msg),
&metrics,
),
}
}
}
NetworkBridgeMessage::SendCollationMessage(peers, msg) => {
gum::trace!(
target: LOG_TARGET,
action = "SendCollationMessages",
num_messages = 1usize,
);
match msg {
Versioned::V1(msg) => send_collation_message_v1(
&mut network_service,
peers,
WireMessage::ProtocolMessage(msg),
&metrics,
),
}
}
NetworkBridgeMessage::SendCollationMessages(msgs) => {
gum::trace!(
target: LOG_TARGET,
action = "SendCollationMessages",
num_messages = %msgs.len(),
);
for (peers, msg) in msgs {
match msg {
Versioned::V1(msg) => send_collation_message_v1(
&mut network_service,
peers,
WireMessage::ProtocolMessage(msg),
&metrics,
),
}
}
}
NetworkBridgeMessage::SendRequests(reqs, if_disconnected) => {
gum::trace!(
target: LOG_TARGET,
action = "SendRequests",
num_requests = %reqs.len(),
);
for req in reqs {
network_service
.start_request(&mut authority_discovery_service, req, if_disconnected)
.await;
}
}
NetworkBridgeMessage::ConnectToValidators {
validator_ids,
peer_set,
failed,
} => {
gum::trace!(
target: LOG_TARGET,
action = "ConnectToValidators",
peer_set = ?peer_set,
ids = ?validator_ids,
"Received a validator connection request",
);
metrics.note_desired_peer_count(peer_set, validator_ids.len());
let (ns, ads) = validator_discovery.on_request(
validator_ids,
peer_set,
failed,
network_service,
authority_discovery_service,
).await;
network_service = ns;
authority_discovery_service = ads;
}
NetworkBridgeMessage::ConnectToResolvedValidators {
validator_addrs,
peer_set,
} => {
gum::trace!(
target: LOG_TARGET,
action = "ConnectToPeers",
peer_set = ?peer_set,
?validator_addrs,
"Received a resolved validator connection request",
);
metrics.note_desired_peer_count(peer_set, validator_addrs.len());
let all_addrs = validator_addrs.into_iter().flatten().collect();
network_service = validator_discovery.on_resolved_request(
all_addrs,
peer_set,
network_service,
).await;
}
NetworkBridgeMessage::NewGossipTopology {
session,
our_neighbors_x,
our_neighbors_y,
} => {
gum::debug!(
target: LOG_TARGET,
action = "NewGossipTopology",
neighbors_x = our_neighbors_x.len(),
neighbors_y = our_neighbors_y.len(),
"Gossip topology has changed",
);
let gossip_peers_x = update_gossip_peers_1d(
&mut authority_discovery_service,
our_neighbors_x,
).await;
let gossip_peers_y = update_gossip_peers_1d(
&mut authority_discovery_service,
our_neighbors_y,
).await;
dispatch_validation_event_to_all_unbounded(
NetworkBridgeEvent::NewGossipTopology(
NewGossipTopology {
session,
our_neighbors_x: gossip_peers_x,
our_neighbors_y: gossip_peers_y,
}
),
ctx.sender(),
);
}
}
Err(e) => return Err(e.into()),
},
}
}
}
async fn update_gossip_peers_1d(
ads: &mut AD,
neighbors: N,
) -> HashMap
where
AD: validator_discovery::AuthorityDiscovery,
N: IntoIterator- ,
N::IntoIter: std::iter::ExactSizeIterator,
{
let neighbors = neighbors.into_iter();
let mut peers = HashMap::with_capacity(neighbors.len());
for (authority, validator_index) in neighbors {
let addr = get_peer_id_by_authority_id(ads, authority.clone()).await;
if let Some(peer_id) = addr {
peers.insert(authority, TopologyPeerInfo { peer_ids: vec![peer_id], validator_index });
}
}
peers
}
async fn handle_network_messages(
mut sender: impl overseer::NetworkBridgeSenderTrait,
mut network_service: impl Network,
network_stream: BoxStream<'static, NetworkEvent>,
mut authority_discovery_service: AD,
metrics: Metrics,
shared: Shared,
) -> Result<(), UnexpectedAbort> {
let mut network_stream = network_stream.fuse();
loop {
match network_stream.next().await {
None => return Err(UnexpectedAbort::EventStreamConcluded),
Some(NetworkEvent::Dht(_)) |
Some(NetworkEvent::SyncConnected { .. }) |
Some(NetworkEvent::SyncDisconnected { .. }) => {},
Some(NetworkEvent::NotificationStreamOpened {
remote: peer,
protocol,
role,
negotiated_fallback,
}) => {
let role = ObservedRole::from(role);
let (peer_set, version) = {
let (peer_set, version) = match PeerSet::try_from_protocol_name(&protocol) {
None => continue,
Some(p) => p,
};
if let Some(fallback) = negotiated_fallback {
match PeerSet::try_from_protocol_name(&fallback) {
None => {
gum::debug!(
target: LOG_TARGET,
fallback = &*fallback,
?peer,
?peer_set,
"Unknown fallback",
);
continue
},
Some((p2, v2)) => {
if p2 != peer_set {
gum::debug!(
target: LOG_TARGET,
fallback = &*fallback,
fallback_peerset = ?p2,
protocol = &*protocol,
peerset = ?peer_set,
"Fallback mismatched peer-set",
);
continue
}
(p2, v2)
},
}
} else {
(peer_set, version)
}
};
gum::debug!(
target: LOG_TARGET,
action = "PeerConnected",
peer_set = ?peer_set,
version,
peer = ?peer,
role = ?role
);
let local_view = {
let mut shared = shared.0.lock();
let peer_map = match peer_set {
PeerSet::Validation => &mut shared.validation_peers,
PeerSet::Collation => &mut shared.collation_peers,
};
match peer_map.entry(peer.clone()) {
hash_map::Entry::Occupied(_) => continue,
hash_map::Entry::Vacant(vacant) => {
vacant.insert(PeerData { view: View::default(), version });
},
}
metrics.on_peer_connected(peer_set, version);
metrics.note_peer_count(peer_set, version, peer_map.len());
shared.local_view.clone().unwrap_or(View::default())
};
let maybe_authority =
authority_discovery_service.get_authority_ids_by_peer_id(peer).await;
match peer_set {
PeerSet::Validation => {
dispatch_validation_events_to_all(
vec![
NetworkBridgeEvent::PeerConnected(
peer.clone(),
role,
1,
maybe_authority,
),
NetworkBridgeEvent::PeerViewChange(peer.clone(), View::default()),
],
&mut sender,
)
.await;
send_message(
&mut network_service,
vec![peer],
PeerSet::Validation,
version,
WireMessage::::ViewUpdate(local_view),
&metrics,
);
},
PeerSet::Collation => {
dispatch_collation_events_to_all(
vec![
NetworkBridgeEvent::PeerConnected(
peer.clone(),
role,
1,
maybe_authority,
),
NetworkBridgeEvent::PeerViewChange(peer.clone(), View::default()),
],
&mut sender,
)
.await;
send_message(
&mut network_service,
vec![peer],
PeerSet::Collation,
version,
WireMessage::::ViewUpdate(local_view),
&metrics,
);
},
}
},
Some(NetworkEvent::NotificationStreamClosed { remote: peer, protocol }) => {
let (peer_set, version) = match PeerSet::try_from_protocol_name(&protocol) {
None => continue,
Some(peer_set) => peer_set,
};
gum::debug!(
target: LOG_TARGET,
action = "PeerDisconnected",
peer_set = ?peer_set,
peer = ?peer
);
let was_connected = {
let mut shared = shared.0.lock();
let peer_map = match peer_set {
PeerSet::Validation => &mut shared.validation_peers,
PeerSet::Collation => &mut shared.collation_peers,
};
let w = peer_map.remove(&peer).is_some();
metrics.on_peer_disconnected(peer_set, version);
metrics.note_peer_count(peer_set, version, peer_map.len());
w
};
if was_connected && version == peer_set.get_default_version() {
match peer_set {
PeerSet::Validation =>
dispatch_validation_event_to_all(
NetworkBridgeEvent::PeerDisconnected(peer),
&mut sender,
)
.await,
PeerSet::Collation =>
dispatch_collation_event_to_all(
NetworkBridgeEvent::PeerDisconnected(peer),
&mut sender,
)
.await,
}
}
},
Some(NetworkEvent::NotificationsReceived { remote, messages }) => {
let expected_versions = {
let mut versions = PerPeerSet::