pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-05-30 21:01:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

rewrite network code to use notifications_protocol APIs from Substrate (#788)

Browse Source 
* extract all network code to legacy submodule

* update references to legacy proto

* skeleton of futures-based protocol

* refactor skeleton to use background task

* rename communication_for to build_table_router

* implement internal message types for validation network

* basic ParachainNetwork and TableRouter implementations

* add some module docs

* remove exit-future from validation

* hack: adapt legacy protocol to lack of exit-future

* generalize RegisteredMessageValidator somewhat

* instantiate and teardown table routers

* clean up RouterInner drop logic

* implement most of the statement import loop

* implement statement loop in async/await

* remove unneeded TODO

* most of the collation skeleton

* send session keys and validator roles

* also send role after status

* use config in startup

* point TODO to issue

* fix test compilation
This commit is contained in:
Robert Habermeier
2020-02-10 15:20:45 +01:00
committed by GitHub
parent 6051a2b272
commit 9b23f3f1f0
21 changed files with 1941 additions and 867 deletions
Download Patch File Download Diff File Expand all files Collapse all files
polkadot/network/src/legacy/gossip/attestation.rs
+266
View File
@@ -0,0 +1,266 @@
// Copyright 2019-2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Gossip messages and structures for dealing with attestations (statements of
//! validity of invalidity on parachain candidates).
//!
//! This follows the same principles as other gossip modules (see parent
//! documentation for more details) by being aware of our current chain
//! heads and accepting only information relative to them. Attestations are localized to
//! relay chain head, so this is easily doable.
//!
//! This module also provides a filter, so we can only broadcast messages to
//! peers that are relevant to chain heads they have advertised.
//!
//! Furthermore, since attestations are bottlenecked by the `Candidate` statement,
//! we only accept attestations which are themselves `Candidate` messages, or reference
//! a `Candidate` we are aware of. Otherwise, it is possible we could be forced to
//! consider an infinite amount of attestations produced by a misbehaving validator.
use sc_network_gossip::{ValidationResult as GossipValidationResult};
use sc_network::ReputationChange;
use polkadot_validation::GenericStatement;
use polkadot_primitives::Hash;
use std::collections::{HashMap, HashSet};
use log::warn;
use crate::legacy::router::attestation_topic;
use super::{cost, benefit, MAX_CHAIN_HEADS, LeavesVec,
ChainContext, Known, MessageValidationData, GossipStatement
};
// knowledge about attestations on a single parent-hash.
#[derive(Default)]
pub(super) struct Knowledge {
candidates: HashSet<Hash>,
}
impl Knowledge {
// whether the peer is aware of a candidate with given hash.
fn is_aware_of(&self, candidate_hash: &Hash) -> bool {
self.candidates.contains(candidate_hash)
}
// note that the peer is aware of a candidate with given hash. this should
// be done after observing an incoming candidate message via gossip.
fn note_aware(&mut self, candidate_hash: Hash) {
self.candidates.insert(candidate_hash);
}
}
#[derive(Default)]
pub(super) struct PeerData {
live: HashMap<Hash, Knowledge>,
}
impl PeerData {
/// Update leaves, returning a list of which leaves are new.
pub(super) fn update_leaves(&mut self, leaves: &LeavesVec) -> LeavesVec {
let mut new = LeavesVec::new();
self.live.retain(|k, _| leaves.contains(k));
for &leaf in leaves {
self.live.entry(leaf).or_insert_with(|| {
new.push(leaf);
Default::default()
});
}
new
}
#[cfg(test)]
pub(super) fn note_aware_under_leaf(&mut self, relay_chain_leaf: &Hash, candidate_hash: Hash) {
if let Some(knowledge) = self.live.get_mut(relay_chain_leaf) {
knowledge.note_aware(candidate_hash);
}
}
pub(super) fn knowledge_at_mut(&mut self, parent_hash: &Hash) -> Option<&mut Knowledge> {
self.live.get_mut(parent_hash)
}
/// Get an iterator over all live leaves of this peer.
pub(super) fn leaves(&self) -> impl Iterator<Item = &Hash> {
self.live.keys()
}
}
/// An impartial view of what topics and data are valid based on attestation session data.
pub(super) struct View {
leaf_work: Vec<(Hash, LeafView)>, // hashes of the best DAG-leaves paired with validation data.
topics: HashMap<Hash, Hash>, // maps topic hashes to block hashes.
}
impl Default for View {
fn default() -> Self {
View {
leaf_work: Vec::with_capacity(MAX_CHAIN_HEADS),
topics: Default::default(),
}
}
}
impl View {
fn leaf_view(&self, relay_chain_leaf: &Hash) -> Option<&LeafView> {
self.leaf_work.iter()
.find_map(|&(ref h, ref leaf)| if h == relay_chain_leaf { Some(leaf) } else { None } )
}
fn leaf_view_mut(&mut self, relay_chain_leaf: &Hash) -> Option<&mut LeafView> {
self.leaf_work.iter_mut()
.find_map(|&mut (ref h, ref mut leaf)| if h == relay_chain_leaf { Some(leaf) } else { None } )
}
/// Get our leaves-set. Guaranteed to have length <= MAX_CHAIN_HEADS.
pub(super) fn neighbor_info<'a>(&'a self) -> impl Iterator<Item=Hash> + 'a + Clone {
self.leaf_work.iter().take(MAX_CHAIN_HEADS).map(|(p, _)| p.clone())
}
/// Note new leaf in our local view and validation data necessary to check signatures
/// of statements issued under this leaf.
///
/// This will be pruned later on a call to `prune_old_leaves`, when this leaf
/// is not a leaf anymore.
pub(super) fn new_local_leaf(&mut self, relay_chain_leaf: Hash, validation_data: MessageValidationData) {
self.leaf_work.push((
relay_chain_leaf,
LeafView {
validation_data,
knowledge: Default::default(),
},
));
self.topics.insert(attestation_topic(relay_chain_leaf), relay_chain_leaf);
}
/// Prune old leaf-work that fails the leaf predicate.
pub(super) fn prune_old_leaves<F: Fn(&Hash) -> bool>(&mut self, is_leaf: F) {
let leaf_work = &mut self.leaf_work;
leaf_work.retain(|&(ref relay_chain_leaf, _)| is_leaf(relay_chain_leaf));
self.topics.retain(|_, v| leaf_work.iter().find(|(p, _)| p == v).is_some());
}
/// Whether a message topic is considered live relative to our view. non-live
/// topics do not pertain to our perceived leaves, and are uninteresting to us.
pub(super) fn is_topic_live(&self, topic: &Hash) -> bool {
self.topics.contains_key(topic)
}
/// The relay-chain block hash corresponding to a topic.
pub(super) fn topic_block(&self, topic: &Hash) -> Option<&Hash> {
self.topics.get(topic)
}
/// Validate the signature on an attestation statement of some kind. Should be done before
/// any repropagation of that statement.
pub(super) fn validate_statement_signature<C: ChainContext + ?Sized>(
&mut self,
message: GossipStatement,
chain: &C,
)
-> (GossipValidationResult<Hash>, ReputationChange)
{
// message must reference one of our chain heads and
// if message is not a `Candidate` we should have the candidate available
// in `attestation_view`.
match self.leaf_view(&message.relay_chain_leaf) {
None => {
let cost = match chain.is_known(&message.relay_chain_leaf) {
Some(Known::Leaf) => {
warn!(
target: "network",
"Leaf block {} not considered live for attestation",
message.relay_chain_leaf,
);
cost::NONE
}
Some(Known::Old) => cost::PAST_MESSAGE,
_ => cost::FUTURE_MESSAGE,
};
(GossipValidationResult::Discard, cost)
}
Some(view) => {
// first check that we are capable of receiving this message
// in a DoS-proof manner.
let benefit = match message.signed_statement.statement {
GenericStatement::Candidate(_) => benefit::NEW_CANDIDATE,
GenericStatement::Valid(ref h) | GenericStatement::Invalid(ref h) => {
if !view.knowledge.is_aware_of(h) {
let cost = cost::ATTESTATION_NO_CANDIDATE;
return (GossipValidationResult::Discard, cost);
}
benefit::NEW_ATTESTATION
}
};
// validate signature.
let res = view.validation_data.check_statement(
&message.relay_chain_leaf,
&message.signed_statement,
);
match res {
Ok(()) => {
let topic = attestation_topic(message.relay_chain_leaf);
(GossipValidationResult::ProcessAndKeep(topic), benefit)
}
Err(()) => (GossipValidationResult::Discard, cost::BAD_SIGNATURE),
}
}
}
}
/// whether it's allowed to send a statement to a peer with given knowledge
/// about the relay parent the statement refers to.
pub(super) fn statement_allowed(
&mut self,
statement: &GossipStatement,
relay_chain_leaf: &Hash,
peer_knowledge: &mut Knowledge,
) -> bool {
let signed = &statement.signed_statement;
match signed.statement {
GenericStatement::Valid(ref h) | GenericStatement::Invalid(ref h) => {
// `valid` and `invalid` statements can only be propagated after
// a candidate message is known by that peer.
peer_knowledge.is_aware_of(h)
}
GenericStatement::Candidate(ref c) => {
// if we are sending a `Candidate` message we should make sure that
// attestation_view and their_view reflects that we know about the candidate.
let hash = c.hash();
peer_knowledge.note_aware(hash);
if let Some(attestation_view) = self.leaf_view_mut(&relay_chain_leaf) {
attestation_view.knowledge.note_aware(hash);
}
// at this point, the peer hasn't seen the message or the candidate
// and has knowledge of the relevant relay-chain parent.
true
}
}
}
}
struct LeafView {
validation_data: MessageValidationData,
knowledge: Knowledge,
}
polkadot/network/src/legacy/gossip/message_routing.rs
+339
View File
@@ -0,0 +1,339 @@
// Copyright 2019-2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Data structures and synchronous logic for ICMP message gossip.
//!
//! The parent-module documentation describes some rationale of the general
//! gossip protocol design.
//!
//! The ICMP message-routing gossip works according to those rationale.
//!
//! In this protocol, we perform work under 4 conditions:
//! ### 1. Upon observation of a new leaf in the block-DAG.
//!
//! We first communicate the best leaves to our neighbors in the gossip graph
//! by the means of a neighbor packet. Then, we query to discover the trie roots
//! of all un-routed message queues from the perspective of each of those leaves.
//!
//! For any trie root in the unrouted set for the new leaf, if we have the corresponding
//! queue, we send it to any peers with the new leaf in their latest advertised set.
//!
//! Which parachain those messages go to and from is unimportant, because this is
//! an everybody-sees-everything style protocol. The only important property is "liveness":
//! that the queue root is un-routed at one of the leaves we perceive to be at the head
//! of the block-DAG.
//!
//! In Substrate gossip, every message is associated with a topic. Typically,
//! many messages are grouped under a single topic. In this gossip system, each queue
//! gets its own topic, which is based on the root hash of the queue. This is because
//! many different chain leaves may have the same queue as un-routed, so it's better than
//! attempting to group message packets by the leaf they appear unrouted at.
//!
//! ### 2. Upon a neighbor packet from a peer.
//!
//! The neighbor packet from a peer should contain perceived chain heads of that peer.
//! If there is any overlap between our perceived chain heads and theirs, we send
//! them any known, un-routed message queue from either set.
//!
//! ### 3. Upon receiving a message queue from a peer.
//!
//! If the message queue is in the un-routed set of one of the latest leaves we've updated to,
//! we accept it and relay to any peers who need that queue as well.
//!
//! If not, we report the peer to the peer-set manager for sending us bad data.
//!
//! ### 4. Periodic Pruning
//!
//! We prune messages that are not un-routed from the view of any leaf and cease
//! to attempt to send them to any peer.
use sp_runtime::traits::{BlakeTwo256, Hash as HashT};
use polkadot_primitives::Hash;
use std::collections::{HashMap, HashSet};
use sp_blockchain::Error as ClientError;
use super::{MAX_CHAIN_HEADS, GossipValidationResult, LeavesVec, ChainContext};
/// Construct a topic for a message queue root deterministically.
pub fn queue_topic(queue_root: Hash) -> Hash {
let mut v = queue_root.as_ref().to_vec();
v.extend(b"message_queue");
BlakeTwo256::hash(&v[..])
}
/// A view of which queue roots are current for a given set of leaves.
#[derive(Default)]
pub struct View {
leaves: LeavesVec,
leaf_topics: HashMap<Hash, HashSet<Hash>>, // leaf_hash -> { topics }
expected_queues: HashMap<Hash, (Hash, bool)>, // topic -> (queue-root, known)
}
impl View {
/// Update the set of current leaves. This is called when we perceive a new bset leaf-set.
pub fn update_leaves<T: ChainContext + ?Sized, I>(&mut self, context: &T, new_leaves: I)
-> Result<(), ClientError>
where I: Iterator<Item=Hash>
{
let new_leaves = new_leaves.take(MAX_CHAIN_HEADS);
let old_leaves = std::mem::replace(&mut self.leaves, new_leaves.collect());
let expected_queues = &mut self.expected_queues;
let leaves = &self.leaves;
self.leaf_topics.retain(|l, topics| {
if leaves.contains(l) { return true }
// prune out all data about old leaves we don't follow anymore.
for topic in topics.iter() {
expected_queues.remove(topic);
}
false
});
let mut res = Ok(());
// add in new data about fresh leaves.
for new_leaf in &self.leaves {
if old_leaves.contains(new_leaf) { continue }
let mut this_leaf_topics = HashSet::new();
let r = context.leaf_unrouted_roots(new_leaf, &mut |&queue_root| {
let topic = queue_topic(queue_root);
this_leaf_topics.insert(topic);
expected_queues.entry(topic).or_insert((queue_root, false));
});
if r.is_err() {
if let Err(e) = res {
log::debug!(target: "message_routing", "Ignored duplicate error {}", e)
};
res = r;
}
self.leaf_topics.insert(*new_leaf, this_leaf_topics);
}
res
}
/// Validate an incoming message queue against this view. If it is accepted
/// by our view of un-routed message queues, we will keep and re-propagate.
pub fn validate_queue_and_note_known(&mut self, messages: &super::GossipParachainMessages)
-> (GossipValidationResult<Hash>, sc_network::ReputationChange)
{
let ostensible_topic = queue_topic(messages.queue_root);
match self.expected_queues.get_mut(&ostensible_topic) {
None => (GossipValidationResult::Discard, super::cost::UNNEEDED_ICMP_MESSAGES),
Some(&mut (_, ref mut known)) => {
if !messages.queue_root_is_correct() {
(
GossipValidationResult::Discard,
super::cost::icmp_messages_root_mismatch(messages.messages.len()),
)
} else {
*known = true;
(
GossipValidationResult::ProcessAndKeep(ostensible_topic),
super::benefit::NEW_ICMP_MESSAGES,
)
}
}
}
}
/// Whether a message with given topic is live.
pub fn is_topic_live(&self, topic: &Hash) -> bool {
self.expected_queues.get(topic).is_some()
}
/// Whether a message is allowed under the intersection of the given leaf-set
/// and our own.
pub fn allowed_intersecting(&self, other_leaves: &LeavesVec, topic: &Hash) -> bool {
for i in other_leaves {
for j in &self.leaves {
if i == j {
let leaf_topics = self.leaf_topics.get(i)
.expect("leaf_topics are mutated only in update_leaves; \
we have an entry for each item in self.leaves; \
i is in self.leaves; qed");
if leaf_topics.contains(topic) {
return true;
}
}
}
}
false
}
/// Get topics of all message queues a peer is interested in - this is useful
/// when a peer has informed us of their new best leaves.
pub fn intersection_topics(&self, other_leaves: &LeavesVec) -> impl Iterator<Item=Hash> {
let deduplicated = other_leaves.iter()
.filter_map(|l| self.leaf_topics.get(l))
.flat_map(|topics| topics.iter().cloned())
.collect::<HashSet<_>>();
deduplicated.into_iter()
}
/// Iterate over all live message queues for which the data is marked as not locally known,
/// calling a closure with `(topic, root)`. The closure will return whether the queue data is
/// unknown.
///
/// This is called when we should send un-routed message queues that we are
/// newly aware of to peers - as in when we update our leaves.
pub fn sweep_unknown_queues(&mut self, mut check_known: impl FnMut(&Hash, &Hash) -> bool) {
for (topic, &mut (ref queue_root, ref mut known)) in self.expected_queues.iter_mut() {
if !*known {
*known = check_known(topic, queue_root)
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::legacy::tests::TestChainContext;
use crate::legacy::gossip::{Known, GossipParachainMessages};
use polkadot_primitives::parachain::Message as ParachainMessage;
fn hash(x: u8) -> Hash {
[x; 32].into()
}
fn message_queue(from: u8, to: u8) -> Option<[[u8; 2]; 1]> {
if from == to {
None
} else {
Some([[from, to]])
}
}
fn message_queue_root(from: u8, to: u8) -> Option<Hash> {
message_queue(from, to).map(
|q| polkadot_validation::message_queue_root(q.iter())
)
}
// check that our view has all of the roots of the message queues
// emitted in the heads identified in `our_heads`, and none of the others.
fn check_roots(view: &mut View, our_heads: &[u8], n_heads: u8) -> bool {
for i in 0..n_heads {
for j in 0..n_heads {
if let Some(messages) = message_queue(i, j) {
let queue_root = message_queue_root(i, j).unwrap();
let messages = GossipParachainMessages {
queue_root,
messages: messages.iter().map(|m| ParachainMessage(m.to_vec())).collect(),
};
let had_queue = match view.validate_queue_and_note_known(&messages).0 {
GossipValidationResult::ProcessAndKeep(topic) => topic == queue_topic(queue_root),
_ => false,
};
if our_heads.contains(&i) != had_queue {
return false
}
}
}
}
true
}
#[test]
fn update_leaves_none_in_common() {
let mut ctx = TestChainContext::default();
let n_heads = 5;
for i in 0..n_heads {
ctx.known_map.insert(hash(i as u8), Known::Leaf);
let messages_out: Vec<_> = (0..n_heads).filter_map(|j| message_queue_root(i, j)).collect();
if !messages_out.is_empty() {
ctx.ingress_roots.insert(hash(i as u8), messages_out);
}
}
// initialize the view with 2 leaves.
let mut view = View::default();
view.update_leaves(
&ctx,
[hash(0), hash(1)].iter().cloned(),
).unwrap();
// we should have all queue roots that were
// un-routed from the perspective of those 2
// leaves and no others.
assert!(check_roots(&mut view, &[0, 1], n_heads));
// after updating to a disjoint set,
// the property that we are aware of all un-routed
// from the perspective of our known leaves should
// remain the same.
view.update_leaves(
&ctx,
[hash(2), hash(3), hash(4)].iter().cloned(),
).unwrap();
assert!(check_roots(&mut view, &[2, 3, 4], n_heads));
}
#[test]
fn update_leaves_overlapping() {
let mut ctx = TestChainContext::default();
let n_heads = 5;
for i in 0..n_heads {
ctx.known_map.insert(hash(i as u8), Known::Leaf);
let messages_out: Vec<_> = (0..n_heads).filter_map(|j| message_queue_root(i, j)).collect();
if !messages_out.is_empty() {
ctx.ingress_roots.insert(hash(i as u8), messages_out);
}
}
let mut view = View::default();
view.update_leaves(
&ctx,
[hash(0), hash(1), hash(2)].iter().cloned(),
).unwrap();
assert!(check_roots(&mut view, &[0, 1, 2], n_heads));
view.update_leaves(
&ctx,
[hash(2), hash(3), hash(4)].iter().cloned(),
).unwrap();
// after updating to a leaf-set overlapping with the prior,
// the property that we are aware of all un-routed
// from the perspective of our known leaves should
// remain the same.
assert!(check_roots(&mut view, &[2, 3, 4], n_heads));
}
}
polkadot/network/src/legacy/gossip/mod.rs
+1417
View File
File diff suppressed because it is too large Load Diff